Compare commits
79
Commits
4a3ebc6b5b
...
31bee2c2fd
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
31bee2c2fd | ||
|
|
bf3e4af6b1 | ||
|
|
f5d9021745 | ||
|
|
633c758d4c | ||
|
|
95ba99ed29 | ||
|
|
944007d9e5 | ||
|
|
db2851e54a | ||
|
|
69aed12cf8 | ||
|
|
86d314050f | ||
|
|
b5f9fdbbac | ||
|
|
72485816d9 | ||
|
|
21fb7de7bf | ||
|
|
010521f10d | ||
|
|
0f72060d66 | ||
|
|
7c0a916c5d | ||
|
|
8c518d4f09 | ||
|
|
4cfd09ebda | ||
|
|
812d662602 | ||
|
|
8918d40b59 | ||
|
|
7e9cd428a1 | ||
|
|
e5e541446f | ||
|
|
2c6efaabe8 | ||
|
|
eb68fb9105 | ||
|
|
cb27582783 | ||
|
|
24bcbc5087 | ||
|
|
5df639bb4e | ||
|
|
ef55e095c3 | ||
|
|
036608b012 | ||
|
|
6c956ee3bd | ||
|
|
eeea036c12 | ||
|
|
a4aecf9ec7 | ||
|
|
3575cdb126 | ||
|
|
e3fcd7e3ae | ||
|
|
7f6055caa2 | ||
|
|
fb52250c11 | ||
|
|
fba504d5bb | ||
|
|
ad80cac06d | ||
|
|
dec8be19de | ||
|
|
281491646d | ||
|
|
bef464a69d | ||
|
|
cff46af086 | ||
|
|
2918e3fda1 | ||
|
|
512ca2fc0b | ||
|
|
63e2d8be20 | ||
|
|
daa56356c8 | ||
|
|
66c3d11016 | ||
|
|
7f4385a422 | ||
|
|
54d4ab1b9a | ||
|
|
09ea3a66e9 | ||
|
|
e2eaefdec9 | ||
|
|
6705330924 | ||
|
|
2e68aba2a7 | ||
|
|
23349e43b2 | ||
|
|
3b5fc7c7be | ||
|
|
025fd08603 | ||
|
|
bc0a1cee82 | ||
|
|
7351759ecf | ||
|
|
f1e5437a3a | ||
|
|
31ad7e113f | ||
|
|
754fb3e29d | ||
|
|
17c1c70447 | ||
|
|
03c8ebae34 | ||
|
|
4c6c7af076 | ||
|
|
c841dce4a4 | ||
|
|
73ab1259c1 | ||
|
|
5de9f4a6ae | ||
|
|
9bc82b7ceb | ||
|
|
5ac158fbe6 | ||
|
|
c646a6c792 | ||
|
|
7118c2f999 | ||
|
|
8ca58f1714 | ||
|
|
b1fdcd6e27 | ||
|
|
62acb3dab8 | ||
|
|
41f6691c26 | ||
|
|
66b3d8c108 | ||
|
|
a902e402c6 | ||
|
|
7e5f658d75 | ||
|
|
461a8a0f82 | ||
|
|
03f5313f0f |
@@ -20,7 +20,8 @@
|
||||
const int32_t *__restrict__ num_tokens_past_padded_ptr, \
|
||||
const float *__restrict__ topk_weights_ptr, int top_k, \
|
||||
bool mul_topk_weights, int prob_m, int prob_n, int prob_k, int *locks, \
|
||||
bool has_bias, bool use_atomic_add, bool use_fp32_reduce
|
||||
bool has_bias, bool use_atomic_add, bool use_fp32_reduce, \
|
||||
bool use_full_k
|
||||
|
||||
namespace MARLIN_NAMESPACE_NAME {
|
||||
template <const vllm::ScalarTypeId a_type_id, // A ScalarType id
|
||||
|
||||
@@ -74,7 +74,8 @@ __global__ void Marlin(
|
||||
int prob_k, // reduction dimension k
|
||||
int* locks, // extra global storage for barrier synchronization
|
||||
bool use_atomic_add, // whether to use atomic add to reduce
|
||||
bool use_fp32_reduce // whether to use fp32 global reduce
|
||||
bool use_fp32_reduce, // whether to use fp32 global reduce
|
||||
bool use_full_k // whether to reduce over full K per block
|
||||
) {}
|
||||
|
||||
} // namespace MARLIN_NAMESPACE_NAME
|
||||
@@ -250,7 +251,8 @@ __global__ void Marlin(
|
||||
int* locks, // extra global storage for barrier synchronization
|
||||
bool has_bias,
|
||||
bool use_atomic_add, // whether to use atomic add to reduce
|
||||
bool use_fp32_reduce // whether to use fp32 global reduce
|
||||
bool use_fp32_reduce, // whether to use fp32 global reduce
|
||||
bool use_full_k // whether to reduce over full K per block
|
||||
) {
|
||||
// Each threadblock processes one "stripe" of the B matrix with (roughly) the
|
||||
// same size, which might involve multiple column "slices" (of width 16 *
|
||||
@@ -370,11 +372,13 @@ __global__ void Marlin(
|
||||
// see https://github.com/vllm-project/vllm/pull/24722 for more details
|
||||
if (global_mn_tiles > gridDim.x) {
|
||||
part2_mn_tiles = global_mn_tiles % gridDim.x;
|
||||
if (part2_mn_tiles * 3 <= gridDim.x) part2_mn_tiles += gridDim.x;
|
||||
if (!use_full_k && part2_mn_tiles * 3 <= gridDim.x)
|
||||
part2_mn_tiles += gridDim.x;
|
||||
part1_mn_iters = (global_mn_tiles - part2_mn_tiles) / gridDim.x;
|
||||
}
|
||||
|
||||
int iters = div_ceil(k_tiles * part2_mn_tiles, gridDim.x);
|
||||
int iters =
|
||||
use_full_k ? k_tiles : div_ceil(k_tiles * part2_mn_tiles, gridDim.x);
|
||||
|
||||
if constexpr (group_blocks != -1) {
|
||||
if (group_blocks >= thread_k_blocks) {
|
||||
|
||||
@@ -33,6 +33,53 @@
|
||||
#include <torch/headeronly/util/Exception.h>
|
||||
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
#include <atomic>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace {
|
||||
// -------------------------------------------------------------------------
|
||||
// MERLIN_FULL_K: the CUDA half of the unmerged upstream PR #46639
|
||||
// (`use_full_k`), which makes the Marlin MoE split-K reduction order
|
||||
// independent of how many blocks the launch was split across.
|
||||
//
|
||||
// Our own toggle rather than upstream's VLLM_BATCH_INVARIANT, because that
|
||||
// one aborts engine startup on GDN models (upstream #42960) and so can never
|
||||
// reach this kernel on Flash-Next.
|
||||
//
|
||||
// Resolved ONCE per process, not per launch, so it is a launch constant in
|
||||
// exactly the sense topk.cu's exact_medium_enabled() is: a cudagraph bakes in
|
||||
// the value that was live at capture time and every replay agrees with it.
|
||||
// Code default off; vllm/env_override.py sets 1 (the shipped value) when unset.
|
||||
//
|
||||
// This std::getenv is the SINGLE source of truth. vllm/envs.py carries a raw
|
||||
// passthrough entry for validate_environ and the compile-factor hash only;
|
||||
// nothing in Python may parse this name a second time.
|
||||
//
|
||||
// The initialiser also
|
||||
// publishes the value it cached into g_cached_full_k (-1 = not initialised
|
||||
// yet; nothing else writes it), and marlin_moe_full_k_state() below only
|
||||
// LOADS it -- asking never fills the cache. Host code only.
|
||||
// -------------------------------------------------------------------------
|
||||
std::atomic<int> g_cached_full_k{-1};
|
||||
|
||||
bool merlin_full_k() {
|
||||
static const bool enabled = [] {
|
||||
const char* raw = std::getenv("MERLIN_FULL_K");
|
||||
const bool value =
|
||||
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
|
||||
g_cached_full_k.store(value ? 1 : 0, std::memory_order_release);
|
||||
return value;
|
||||
}();
|
||||
return enabled;
|
||||
}
|
||||
|
||||
// MERLIN_FULL_K as this process cached it: -1 = not initialised yet (no Marlin
|
||||
// MoE launch has happened in this process), 0 = off, 1 = on. Tensor-free, no
|
||||
// CUDA call.
|
||||
int64_t marlin_moe_full_k_state() {
|
||||
return static_cast<int64_t>(g_cached_full_k.load(std::memory_order_acquire));
|
||||
}
|
||||
} // namespace
|
||||
|
||||
#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
|
||||
static_assert(std::is_same<scalar_t, half>::value || \
|
||||
@@ -183,7 +230,8 @@ exec_config_t determine_exec_config(
|
||||
const vllm::ScalarType& c_type, const vllm::ScalarType& s_type, int prob_m,
|
||||
int prob_n, int prob_k, int num_experts, int top_k, int thread_m_blocks,
|
||||
bool m_block_size_8, int num_bits, int group_size, bool has_zp,
|
||||
bool is_zp_float, bool is_a_8bit, int stages, int max_shared_mem, int sms) {
|
||||
bool is_zp_float, bool is_a_8bit, int stages, int max_shared_mem, int sms,
|
||||
bool use_full_k) {
|
||||
exec_config_t exec_cfg = exec_config_t{1, thread_config_t{-1, -1, -1}};
|
||||
thread_config_t* thread_configs = thread_m_blocks > 1
|
||||
? large_batch_thread_configs
|
||||
@@ -228,7 +276,8 @@ exec_config_t determine_exec_config(
|
||||
else
|
||||
allow_count = max(min(allow_count, 2), 1);
|
||||
|
||||
if (prob_n / th_config.thread_n * prob_m * top_k * 4 < sms * allow_count) {
|
||||
if (!use_full_k &&
|
||||
prob_n / th_config.thread_n * prob_m * top_k * 4 < sms * allow_count) {
|
||||
allow_count =
|
||||
max(prob_n / th_config.thread_n * prob_m * top_k * 4 / sms, 1);
|
||||
}
|
||||
@@ -254,6 +303,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
int group_size, int dev, cudaStream_t stream, int thread_k,
|
||||
int thread_n, int sms, int blocks_per_sm, bool use_atomic_add,
|
||||
bool use_fp32_reduce, bool is_zp_float) {
|
||||
const bool use_full_k = merlin_full_k();
|
||||
int thread_m_blocks = div_ceil(moe_block_size, 16);
|
||||
bool m_block_size_8 = moe_block_size == 8;
|
||||
bool is_a_8bit = a_type.size_bits() == 8;
|
||||
@@ -331,7 +381,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
exec_cfg = determine_exec_config(
|
||||
a_type, b_type, c_type, s_type, prob_m, prob_n, prob_k, num_experts,
|
||||
top_k, thread_m_blocks, m_block_size_8, num_bits, group_size, has_zp,
|
||||
is_zp_float, is_a_8bit, stages, max_shared_mem, sms);
|
||||
is_zp_float, is_a_8bit, stages, max_shared_mem, sms, use_full_k);
|
||||
thread_tfg = exec_cfg.tb_cfg;
|
||||
}
|
||||
|
||||
@@ -383,7 +433,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
A_ptr, B_ptr, C_ptr, C_tmp_ptr, bias_ptr, a_s_ptr, b_s_ptr, g_s_ptr, zp_ptr,
|
||||
sorted_token_ids_ptr, expert_ids_ptr, num_tokens_past_padded_ptr,
|
||||
topk_weights_ptr, top_k, mul_topk_weights, prob_m, prob_n, prob_k, locks,
|
||||
has_bias, use_atomic_add, use_fp32_reduce);
|
||||
has_bias, use_atomic_add, use_fp32_reduce, use_full_k);
|
||||
// clang-format on
|
||||
}
|
||||
|
||||
@@ -704,3 +754,14 @@ torch::stable::Tensor moe_wna16_marlin_gemm(
|
||||
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
|
||||
m.impl("moe_wna16_marlin_gemm", TORCH_BOX(&moe_wna16_marlin_gemm));
|
||||
}
|
||||
|
||||
// v2.2 -- defined and registered in this TU so the query exists exactly when
|
||||
// the static it reads does. Tensor-free, so CompositeExplicitAutograd (a CUDA
|
||||
// registration could never be dispatched to without a tensor argument).
|
||||
STABLE_TORCH_LIBRARY_FRAGMENT(_moe_C, m) {
|
||||
m.def("marlin_moe_full_k_state() -> int");
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CompositeExplicitAutograd, m) {
|
||||
m.impl("marlin_moe_full_k_state", TORCH_BOX(&marlin_moe_full_k_state));
|
||||
}
|
||||
|
||||
@@ -481,6 +481,18 @@ void persistent_topk(const torch::stable::Tensor& logits,
|
||||
torch::stable::Tensor& workspace, int64_t k,
|
||||
int64_t max_seq_len);
|
||||
|
||||
// Medium-path exact-restart counters; see topk.cu for the slot layout.
|
||||
void persistent_topk_exact_counters(torch::stable::Tensor& out, int64_t reset,
|
||||
int64_t sync);
|
||||
|
||||
// The slot count of THIS build's counter array, so the Python
|
||||
// reader never has to guess it from an environment variable.
|
||||
int64_t persistent_topk_exact_counters_slots();
|
||||
|
||||
// The persistent_topk switches as this
|
||||
// process cached them, packed; see topk.cu for the layout.
|
||||
int64_t persistent_topk_toggle_state();
|
||||
|
||||
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
||||
void cooperative_topk(const torch::stable::Tensor& logits,
|
||||
const torch::stable::Tensor& lengths,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,6 +3,11 @@
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <mutex>
|
||||
|
||||
#include "ops.h"
|
||||
#include "torch_utils.h"
|
||||
@@ -15,6 +20,388 @@
|
||||
namespace {
|
||||
|
||||
#ifndef USE_ROCM
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MERLIN_TOPK_EXACT_MEDIUM switch and its counters.
|
||||
//
|
||||
// The switch is resolved ONCE per process, not per launch, so it is a launch
|
||||
// constant: a cudagraph captures the value that was live at capture time and
|
||||
// every replay agrees with it. Default ON in this release; set
|
||||
// MERLIN_TOPK_EXACT_MEDIUM=0 for the A/B control arm, which restores v2.0.1
|
||||
// behaviour exactly (the counters still record, so the control arm reports the
|
||||
// overflow incidence it is the control for).
|
||||
// ---------------------------------------------------------------------------
|
||||
// -------------------------------------------------------------------------
|
||||
// MERLIN_TIE_DET: deterministic tie-break at the k-th cutoff in
|
||||
// persistent_topk. Resolved once per process, exactly like
|
||||
// exact_medium_enabled() below, so it is a launch constant that a cudagraph
|
||||
// bakes in at capture.
|
||||
//
|
||||
// Our own toggle rather than upstream's VLLM_BATCH_INVARIANT, which aborts
|
||||
// engine startup on GDN models (upstream #42960) and so can never reach this
|
||||
// kernel on Flash-Next. Code default off; vllm/env_override.py sets 1 (the shipped value) when unset.
|
||||
//
|
||||
// This std::getenv is the SINGLE source of truth; vllm/envs.py carries a raw
|
||||
// passthrough for validate_environ and the compile-factor hash only.
|
||||
// -------------------------------------------------------------------------
|
||||
// -------------------------------------------------------------------------
|
||||
// A READ-ONLY MIRROR of the
|
||||
// switches as this process CACHED them, for persistent_topk_toggle_state().
|
||||
//
|
||||
// Each switch below is a function-local static filled once by its lambda. The
|
||||
// lambda now also stores the value it cached into the matching atomic here
|
||||
// (publish_cached), and nothing else ever writes these atomics, so a mirror
|
||||
// either says "not initialised yet" (-1) or holds exactly the cached value.
|
||||
// The query only LOADS them: it never calls a getter, so asking cannot fill a
|
||||
// cache from an environment that differs from the one the kernel saw. The
|
||||
// parse expressions are unchanged; this is host code only.
|
||||
// -------------------------------------------------------------------------
|
||||
std::atomic<int> g_cached_tie_recent{-1};
|
||||
std::atomic<int> g_cached_tie_det{-1};
|
||||
std::atomic<int> g_cached_exact_medium{-1};
|
||||
std::atomic<int> g_cached_det_large{-1};
|
||||
std::atomic<int> g_cached_exact_decode{-1};
|
||||
std::atomic<int> g_cached_sorted_emit{-1};
|
||||
|
||||
bool publish_cached(std::atomic<int>& mirror, bool value) {
|
||||
mirror.store(value ? 1 : 0, std::memory_order_release);
|
||||
return value;
|
||||
}
|
||||
|
||||
// MERLIN_TIE_RECENT -- the DIRECTION of the deterministic tie-break above.
|
||||
// Measured at 12k tokens on the reference host: oldest-wins costs -7.9 % decode,
|
||||
// most-recent-wins is +1.5 %. Both deterministic. Also the direction
|
||||
// medium_exact_restart() already implements, so setting this alongside
|
||||
// MERLIN_TIE_DET makes the medium path agree with itself on overflowing
|
||||
// rows. Only read when MERLIN_TIE_DET is set; code default OFF; vllm/env_override.py sets 1 (the shipped value) when unset.
|
||||
bool tie_recent_enabled() {
|
||||
static const bool enabled = [] {
|
||||
const char* raw = std::getenv("MERLIN_TIE_RECENT");
|
||||
return publish_cached(
|
||||
g_cached_tie_recent,
|
||||
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
|
||||
}();
|
||||
return enabled;
|
||||
}
|
||||
|
||||
bool det_ties_enabled() {
|
||||
static const bool enabled = [] {
|
||||
const char* raw = std::getenv("MERLIN_TIE_DET");
|
||||
return publish_cached(
|
||||
g_cached_tie_det,
|
||||
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
|
||||
}();
|
||||
return enabled;
|
||||
}
|
||||
|
||||
bool exact_medium_enabled() {
|
||||
static const bool enabled = [] {
|
||||
const char* raw = std::getenv("MERLIN_TOPK_EXACT_MEDIUM");
|
||||
return publish_cached(
|
||||
g_cached_exact_medium,
|
||||
(raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0));
|
||||
}();
|
||||
return enabled;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MERLIN_TOPK_DET_LARGE switch.
|
||||
//
|
||||
// Same contract as the medium-path switch: resolved once per process, so it is a
|
||||
// launch constant and a cudagraph captures one value. Default ON on this
|
||||
// branch. MERLIN_TOPK_DET_LARGE=0 is the A/B control arm and runs the
|
||||
// v2.0.1 `radix_topk` tail verbatim -- the deterministic emission sits behind
|
||||
// one `if (det_large)` in front of it and touches nothing when off, including
|
||||
// the large-path counters, which is what makes the control arm the real baseline.
|
||||
// ---------------------------------------------------------------------------
|
||||
bool det_large_enabled() {
|
||||
static const bool enabled = [] {
|
||||
const char* raw = std::getenv("MERLIN_TOPK_DET_LARGE");
|
||||
return publish_cached(
|
||||
g_cached_det_large,
|
||||
(raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0));
|
||||
}();
|
||||
return enabled;
|
||||
}
|
||||
|
||||
// MERLIN_TOPK_EXACT_DECODE: repair an overflowing decode
|
||||
// row with the same exact restart the medium path uses. Resolved once per
|
||||
// process like the others, so it is a launch constant a cudagraph bakes in.
|
||||
//
|
||||
// Default OFF, unlike MERLIN_TOPK_EXACT_MEDIUM. The medium path's overflow
|
||||
// incidence was measured (0.1-0.2 % of medium rows at 38-51K) before that
|
||||
// switch defaulted on; the decode path's has never been measured at all,
|
||||
// which is precisely what the [10] counter now exists to establish. Detection
|
||||
// is unconditional and free; the repair waits on a number.
|
||||
bool exact_decode_enabled() {
|
||||
static const bool enabled = [] {
|
||||
const char* raw = std::getenv("MERLIN_TOPK_EXACT_DECODE");
|
||||
return publish_cached(
|
||||
g_cached_exact_decode,
|
||||
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
|
||||
}();
|
||||
return enabled;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MERLIN_TOPK_SORTED_EMIT: every persistent_topk
|
||||
// row leaves the op in the canonical order the Python MERLIN_QSA_SORT imposes
|
||||
// (ascending index, padding last as -1), so qsa_indexer.py can drop that sort.
|
||||
// See sorted_emit_row() in persistent_topk.cuh for the contract; the selected
|
||||
// SET is never touched. Resolved once per process like every switch here, so
|
||||
// it is a launch constant a cudagraph bakes in. Code default OFF; vllm/env_override.py
|
||||
// sets 1 (the shipped value) when unset.
|
||||
//
|
||||
// This std::getenv is the single source of truth. Python does not parse the
|
||||
// variable to decide anything: qsa_indexer.py asks THIS cache, through
|
||||
// persistent_topk_toggle_state() field 6, after its first launch -- so the
|
||||
// Python sort is skipped only when the kernel that ran says it sorted.
|
||||
// ---------------------------------------------------------------------------
|
||||
bool sorted_emit_enabled() {
|
||||
static const bool enabled = [] {
|
||||
const char* raw = std::getenv("MERLIN_TOPK_SORTED_EMIT");
|
||||
return publish_cached(
|
||||
g_cached_sorted_emit,
|
||||
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
|
||||
}();
|
||||
return enabled;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MERLIN_TOPK_LARGE_FAULT_INJECT. TEST ONLY.
|
||||
//
|
||||
// -1 (the default, and the value of an unset or negative variable) is off.
|
||||
// Any other value names the `cta_in_group` whose CTA-LOCAL fault predicate the
|
||||
// large path forces, which is the only way to reach the fail-closed arm: no
|
||||
// legal input can make a CTA's own pivot-equal count disagree with the one it
|
||||
// published, so those branches are otherwise untestable (as with the
|
||||
// medium-path restart's own fault path). Resolved once per process, so
|
||||
// it is a launch constant and a cudagraph captures one value. Setting it also
|
||||
// suppresses __trap(), because the point of injecting is to OBSERVE the
|
||||
// trap-off row-failure policy (all -1 row, counter += 1) without a second
|
||||
// build.
|
||||
// ---------------------------------------------------------------------------
|
||||
int large_fault_inject_cta() {
|
||||
static const int cta = [] {
|
||||
const char* raw = std::getenv("MERLIN_TOPK_LARGE_FAULT_INJECT");
|
||||
if (raw == nullptr || raw[0] == '\0') return -1;
|
||||
const int v = std::atoi(raw);
|
||||
return (v < 0) ? -1 : v;
|
||||
}();
|
||||
return cta;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MERLIN_TOPK_LARGE_FAULT_KIND. TEST ONLY, companion to
|
||||
// the variable above; ignored unless MERLIN_TOPK_LARGE_FAULT_INJECT names a
|
||||
// CTA.
|
||||
//
|
||||
// unset / "pre" (default) force the PRE-EMISSION predicate, caught at X3.
|
||||
// "emit" force the EMISSION-BOUND slot check, caught at X4.
|
||||
//
|
||||
// The two are mutually exclusive by construction: a pre-emission fault makes
|
||||
// `row_fault` nonzero, the emission block is skipped wholesale, and the X4
|
||||
// code -- including `suffix_sum[6]`, the word whose concurrent non-atomic
|
||||
// write this change is about -- is never reached. The shipped injector could
|
||||
// therefore only ever exercise the arm that was already correct, which is why
|
||||
// the X4 arm had never been exercised. Resolved once per process like
|
||||
// every other switch here, so it is a launch constant a cudagraph bakes in.
|
||||
// ---------------------------------------------------------------------------
|
||||
bool large_fault_inject_emit() {
|
||||
static const bool emit = [] {
|
||||
const char* raw = std::getenv("MERLIN_TOPK_LARGE_FAULT_KIND");
|
||||
if (raw == nullptr || raw[0] == '\0') return false;
|
||||
return std::strncmp(raw, "emit", 5) == 0;
|
||||
}();
|
||||
return emit;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Say the effective tie policy out loud, once.
|
||||
//
|
||||
// "TIE_DET=1 TIE_RECENT=0" used to mean three different things on the three
|
||||
// paths, and nothing printed the difference. One stderr line at the first
|
||||
// launch of the process naming the policy the kernel will actually apply on
|
||||
// EVERY path. It is a plain fprintf on the host: no CUDA call, so it is safe
|
||||
// from a capturing thread, and the `static` initialiser makes it exactly once
|
||||
// per process regardless of how many workers share the log.
|
||||
//
|
||||
// The line covers all FIVE paths. The large
|
||||
// path is no longer a fixed rule: `radix_emit_deterministic` takes the same
|
||||
// `effective_tie_recent()` value as the medium restart, so `large=` reports
|
||||
// the real direction when MERLIN_TOPK_DET_LARGE is on and the inherited
|
||||
// v2.0.1 atomic-arrival race when it is off. The flag printed here is the
|
||||
// ENV value; a launch whose ctas_per_group overflows the staging row degrades
|
||||
// to atomic-arrival for that launch and leaves counter [4] at 0, which is the
|
||||
// separate warning below.
|
||||
// ---------------------------------------------------------------------------
|
||||
void report_tie_policy_once() {
|
||||
static const bool reported = [] {
|
||||
const bool det = det_ties_enabled();
|
||||
const bool recent =
|
||||
vllm::persistent::effective_tie_recent(det, tie_recent_enabled());
|
||||
const char* dir = recent ? "most-recent-wins" : "oldest-wins";
|
||||
std::fprintf(
|
||||
stderr,
|
||||
"[merlin] persistent_topk tie policy: MERLIN_TIE_DET=%d "
|
||||
"MERLIN_TIE_RECENT=%d MERLIN_TOPK_EXACT_MEDIUM=%d "
|
||||
"MERLIN_TOPK_EXACT_DECODE=%d MERLIN_TOPK_DET_LARGE=%d "
|
||||
"MERLIN_TOPK_SORTED_EMIT=%d -> "
|
||||
"decode-buffered=%s decode-restart=%s medium-buffered=%s "
|
||||
"medium-restart=%s large=%s emission=%s\n",
|
||||
det ? 1 : 0, tie_recent_enabled() ? 1 : 0,
|
||||
exact_medium_enabled() ? 1 : 0, exact_decode_enabled() ? 1 : 0,
|
||||
det_large_enabled() ? 1 : 0, sorted_emit_enabled() ? 1 : 0,
|
||||
det ? dir : "atomic-arrival",
|
||||
exact_decode_enabled() ? dir : "n/a (detect-only; counter [10])",
|
||||
det ? dir : "atomic-arrival",
|
||||
exact_medium_enabled() ? dir : "n/a (exact restart disabled)",
|
||||
det_large_enabled() ? dir : "atomic-arrival (non-deterministic)",
|
||||
sorted_emit_enabled()
|
||||
? "ascending-in-kernel (Python QSA sort skipped)"
|
||||
: "arrival-order on buffered paths (Python QSA sort decides)");
|
||||
return true;
|
||||
}();
|
||||
(void)reported;
|
||||
}
|
||||
|
||||
#if MERLIN_TOPK_DIAG_ANY
|
||||
// DIAGNOSTIC BUILD ONLY (persistent_topk.cuh, "DIAGNOSTIC BUILDS ONLY"). Say
|
||||
// so once per process, on stderr, before the first launch: the diagnostic-build tests
|
||||
// key on this line to prove which build they ran against, and the smoke's
|
||||
// startup-warning scan must reject any process that prints it.
|
||||
void report_diag_build_once() {
|
||||
static const bool reported = [] {
|
||||
std::fprintf(stderr,
|
||||
"[merlin] persistent_topk DIAGNOSTIC BUILD: "
|
||||
"MERLIN_TOPK_DIAG_DELAY=%lld MERLIN_TOPK_DIAG_PREREPAIR=%d "
|
||||
"MERLIN_TOPK_DIAG_FORCE_FAULT=%d "
|
||||
"MERLIN_TOPK_DIAG_FORCE_FAULT_CTA=%d -- NOT FOR SERVING\n",
|
||||
static_cast<long long>(MERLIN_TOPK_DIAG_DELAY),
|
||||
static_cast<int>(MERLIN_TOPK_DIAG_PREREPAIR),
|
||||
static_cast<int>(MERLIN_TOPK_DIAG_FORCE_FAULT),
|
||||
static_cast<int>(MERLIN_TOPK_DIAG_FORCE_FAULT_CTA));
|
||||
return true;
|
||||
}();
|
||||
(void)reported;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Process-lifetime device counters, one block per device. The kernel only
|
||||
// ever atomicAdds into them, so the launch path needs no host sync and the
|
||||
// values accumulate across cudagraph replays.
|
||||
//
|
||||
// Allocation, however, is NOT capture-safe: cudaMalloc during stream capture
|
||||
// is at best not captured and at worst invalidates the capture. So:
|
||||
// - the allocation only ever happens on a call that is explicitly allowed to
|
||||
// allocate, which is a non-capturing launch or the reader op;
|
||||
// - a call made *during* capture never allocates and never latches a result,
|
||||
// so a capture that happens to be first does not permanently poison the
|
||||
// instrument;
|
||||
// - the launch path fails loudly if it is capturing with no counters, rather
|
||||
// than baking a graph that runs the patched kernel with no instrument at
|
||||
// all. A silently uninstrumented graph is the failure this is here to
|
||||
// prevent, so the check is deliberately not a warning.
|
||||
// Read (and, on the first call, allocate) with
|
||||
// torch.ops._C.persistent_topk_exact_counters, which is the explicit
|
||||
// pre-capture init path.
|
||||
constexpr int kMaxCounterDevices = 16;
|
||||
|
||||
enum CounterState : int {
|
||||
kCountersUninit = 0,
|
||||
kCountersReady = 1,
|
||||
kCountersFailed = 2,
|
||||
};
|
||||
|
||||
// Atomics, not plain globals: the fast path reads these without the mutex, and
|
||||
// a second host thread may be publishing them from the slow path. Release on
|
||||
// publish, acquire on read, and the pointer is always published BEFORE the
|
||||
// state that advertises it.
|
||||
std::atomic<unsigned long long*> g_exact_counters[kMaxCounterDevices] = {};
|
||||
std::atomic<int> g_exact_counters_state[kMaxCounterDevices] = {};
|
||||
std::mutex g_exact_counters_mutex;
|
||||
|
||||
// Lookup only. Never allocates, never blocks, safe from any context.
|
||||
unsigned long long* get_exact_counters(int device_index) {
|
||||
if (device_index < 0 || device_index >= kMaxCounterDevices) return nullptr;
|
||||
if (g_exact_counters_state[device_index].load(std::memory_order_acquire) !=
|
||||
kCountersReady) {
|
||||
return nullptr;
|
||||
}
|
||||
return g_exact_counters[device_index].load(std::memory_order_acquire);
|
||||
}
|
||||
|
||||
// Allocate, zero, and publish. MUST NOT be called on a capturing stream, and
|
||||
// should be called in a quiescent lifecycle phase: it allocates and blocks,
|
||||
// and neither is safe while any thread in the process is capturing a graph.
|
||||
// `stream_is_capturing` only proves the CALLING stream is idle of capture, so
|
||||
// the intended entry point is the reader op called explicitly before capture;
|
||||
// the non-capturing launch path calls this as a fallback.
|
||||
unsigned long long* ensure_exact_counters(int device_index, cudaStream_t stream) {
|
||||
if (device_index < 0 || device_index >= kMaxCounterDevices) return nullptr;
|
||||
|
||||
const int state =
|
||||
g_exact_counters_state[device_index].load(std::memory_order_acquire);
|
||||
if (state == kCountersReady) {
|
||||
return g_exact_counters[device_index].load(std::memory_order_acquire);
|
||||
}
|
||||
// A previous attempt failed for real (OOM); do not retry on every launch.
|
||||
// Only a call that was actually allowed to allocate can reach this state, so
|
||||
// a capture-time call never latches it.
|
||||
if (state == kCountersFailed) return nullptr;
|
||||
|
||||
std::lock_guard<std::mutex> lock(g_exact_counters_mutex);
|
||||
// Re-check under the lock; another thread may have published meanwhile.
|
||||
const int locked_state =
|
||||
g_exact_counters_state[device_index].load(std::memory_order_relaxed);
|
||||
if (locked_state != kCountersUninit) {
|
||||
return (locked_state == kCountersReady)
|
||||
? g_exact_counters[device_index].load(std::memory_order_acquire)
|
||||
: nullptr;
|
||||
}
|
||||
|
||||
constexpr size_t bytes =
|
||||
vllm::persistent::kNumTopKExactCounters * sizeof(unsigned long long);
|
||||
unsigned long long* ptr = nullptr;
|
||||
if (cudaMalloc(&ptr, bytes) == cudaSuccess && ptr != nullptr) {
|
||||
// Host acquire/release orders the HOST state word; it says
|
||||
// nothing about the device. cudaMemset on device memory is asynchronous
|
||||
// with respect to the host, so publishing Ready straight after it would
|
||||
// let a first launch -- on this or any other stream -- increment counters
|
||||
// that had not been zeroed yet, or read uninitialised bytes. Zero on a
|
||||
// known stream and WAIT for it, so initialisation is complete before the
|
||||
// pointer becomes visible to any consumer.
|
||||
if (cudaMemsetAsync(ptr, 0, bytes, stream) != cudaSuccess ||
|
||||
cudaStreamSynchronize(stream) != cudaSuccess) {
|
||||
cudaFree(ptr);
|
||||
ptr = nullptr;
|
||||
}
|
||||
} else {
|
||||
ptr = nullptr;
|
||||
}
|
||||
// A failed allocation leaves the error state set; clear it so the launch's
|
||||
// own cudaGetLastError() gate does not report this as a kernel failure.
|
||||
if (ptr == nullptr) cudaGetLastError();
|
||||
|
||||
g_exact_counters[device_index].store(ptr, std::memory_order_release);
|
||||
g_exact_counters_state[device_index].store(
|
||||
ptr != nullptr ? kCountersReady : kCountersFailed,
|
||||
std::memory_order_release);
|
||||
return ptr;
|
||||
}
|
||||
|
||||
// True when the stream is mid-capture, in which case nothing on this path may
|
||||
// allocate. A query failure is treated as "capturing": the conservative
|
||||
// direction is to refuse to allocate, not to risk invalidating a capture.
|
||||
bool stream_is_capturing(cudaStream_t stream) {
|
||||
cudaStreamCaptureStatus status = cudaStreamCaptureStatusNone;
|
||||
if (cudaStreamIsCapturing(stream, &status) != cudaSuccess) {
|
||||
cudaGetLastError(); // do not leak this into the launch's error gate
|
||||
return true;
|
||||
}
|
||||
return status != cudaStreamCaptureStatusNone;
|
||||
}
|
||||
|
||||
template <int TopK>
|
||||
void launch_persistent_topk(const torch::stable::Tensor& logits,
|
||||
const torch::stable::Tensor& lengths,
|
||||
@@ -37,6 +424,19 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
||||
max_smem_per_block = device_prop->sharedMemPerBlockOptin;
|
||||
}
|
||||
|
||||
// MERLIN_TOPK_SORTED_EMIT on the two fallbacks that bypass
|
||||
// persistent_topk_kernel: re-emit every row ascending on the same stream.
|
||||
// A plain launch with a fixed configuration, so it is capture-safe.
|
||||
const auto sorted_emit_fallback_rows = [&]() {
|
||||
if (!sorted_emit_enabled()) return;
|
||||
report_tie_policy_once();
|
||||
P::sorted_emit_rows_kernel<TopK>
|
||||
<<<static_cast<unsigned>(num_rows), P::kThreadsPerBlock,
|
||||
P::sorted_emit_smem_bytes<TopK>(), stream>>>(
|
||||
output.mutable_data_ptr<int32_t>(),
|
||||
static_cast<uint32_t>(num_rows));
|
||||
};
|
||||
|
||||
// Allow static fallback storage in addition to the 128 KiB dynamic buffer.
|
||||
if (num_rows > 64 &&
|
||||
max_seq_len >= vllm::sampled_topk::kMinSampledLength<TopK> &&
|
||||
@@ -52,6 +452,7 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
||||
logits.const_data_ptr<float>(), lengths.const_data_ptr<int32_t>(),
|
||||
output.mutable_data_ptr<int32_t>(), stride,
|
||||
static_cast<int>(std::min(max_seq_len, logits.size(1))));
|
||||
sorted_emit_fallback_rows();
|
||||
} else if (num_rows > 32 && max_smem_per_block >= 128 * 1024) {
|
||||
cudaError_t status =
|
||||
vllm::FilteredTopKRaggedTransform<float, int32_t, TopK>(
|
||||
@@ -60,6 +461,7 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
||||
static_cast<uint32_t>(TopK), static_cast<uint32_t>(stride), stream);
|
||||
STD_TORCH_CHECK(status == cudaSuccess,
|
||||
"FilteredTopK failed: ", cudaGetErrorString(status));
|
||||
sorted_emit_fallback_rows();
|
||||
} else {
|
||||
STD_TORCH_CHECK(workspace.is_cuda(), "workspace must be CUDA tensor");
|
||||
STD_TORCH_CHECK(
|
||||
@@ -156,6 +558,10 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
||||
const int64_t next_n = lengths.dim() == 2 ? lengths.size(1) : 1;
|
||||
top_k_per_row_decode(logits, next_n, lengths, output, num_rows,
|
||||
logits.stride(0), logits.stride(1), TopK);
|
||||
sorted_emit_fallback_rows();
|
||||
const cudaError_t fb_err = cudaGetLastError();
|
||||
STD_TORCH_CHECK(fb_err == cudaSuccess, "sorted emission failed: ",
|
||||
cudaGetErrorString(fb_err));
|
||||
return;
|
||||
}
|
||||
cudaError_t status =
|
||||
@@ -167,6 +573,10 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
||||
static_cast<uint32_t>(stride), stream);
|
||||
STD_TORCH_CHECK(status == cudaSuccess, "FilteredTopK fallback failed: ",
|
||||
cudaGetErrorString(status));
|
||||
sorted_emit_fallback_rows();
|
||||
const cudaError_t fb_err = cudaGetLastError();
|
||||
STD_TORCH_CHECK(fb_err == cudaSuccess, "sorted emission failed: ",
|
||||
cudaGetErrorString(fb_err));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -218,6 +628,81 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
||||
workspace.mutable_data_ptr<uint8_t>());
|
||||
params.ctas_per_group = ctas_per_group;
|
||||
params.max_seq_len = static_cast<uint32_t>(max_seq_len);
|
||||
params.exact_medium = exact_medium_enabled();
|
||||
// The per-CTA selected / pivot-equal counts are staged in ONE
|
||||
// 256-slot RadixRowState histogram row (the slot round 3 has just finished
|
||||
// reading). Two more per-CTA regions were added for the
|
||||
// group-uniform fault verdicts, so the row now holds 4 * ctas_per_group
|
||||
// uint32s. That is still the whole reason the workspace contract is
|
||||
// unchanged, and it is also the only
|
||||
// limit: above 64 CTAs per group the deterministic path cannot be
|
||||
// expressed there, so the launch degrades to the v2.0.1 tail rather than
|
||||
// failing. Unreachable at any shape this kernel serves (ctas_per_group is
|
||||
// 2-3 across our 135K-262K band and <= 16 at the smallest chunk size), but
|
||||
// a silent loss of determinism is exactly what this branch exists to stop,
|
||||
// so say it once.
|
||||
const bool det_large_fits =
|
||||
4u * ctas_per_group <= static_cast<uint32_t>(P::RADIX);
|
||||
params.det_large = det_large_enabled() && det_large_fits;
|
||||
params.fault_inject_cta = large_fault_inject_cta();
|
||||
params.fault_inject_emit = large_fault_inject_emit();
|
||||
if (det_large_enabled() && !det_large_fits) {
|
||||
static std::atomic<bool> det_large_warned{false};
|
||||
if (!det_large_warned.exchange(true)) {
|
||||
std::fprintf(stderr,
|
||||
"[vllm] MERLIN_TOPK_DET_LARGE is on but ctas_per_group=%u "
|
||||
"needs %u of %d staging slots; the large path falls back "
|
||||
"to nondeterministic placement at this shape (counter [4] "
|
||||
"stays 0).\n",
|
||||
ctas_per_group, 4u * ctas_per_group, P::RADIX);
|
||||
}
|
||||
}
|
||||
if (params.fault_inject_cta >= 0) {
|
||||
static std::atomic<bool> inject_warned{false};
|
||||
if (!inject_warned.exchange(true)) {
|
||||
std::fprintf(stderr,
|
||||
"[vllm] MERLIN_TOPK_LARGE_FAULT_INJECT=%d kind=%s is "
|
||||
"set: the large path FORCES a %s fault in that CTA of "
|
||||
"every deterministic large row and returns an all -1 row. "
|
||||
"This is a test mode and must not be set in "
|
||||
"production.\n",
|
||||
params.fault_inject_cta,
|
||||
params.fault_inject_emit ? "emit" : "pre",
|
||||
params.fault_inject_emit ? "emission-bound (X4)"
|
||||
: "pre-emission (X3)");
|
||||
}
|
||||
}
|
||||
params.det_ties = det_ties_enabled();
|
||||
params.tie_recent = tie_recent_enabled();
|
||||
params.exact_decode = exact_decode_enabled();
|
||||
params.sorted_emit = sorted_emit_enabled();
|
||||
report_tie_policy_once();
|
||||
#if MERLIN_TOPK_DIAG_ANY
|
||||
report_diag_build_once();
|
||||
#endif
|
||||
// Allocate only off the capture path; in the ordinary engine lifecycle the
|
||||
// eager warmup lands here first, long before capture. The explicit,
|
||||
// quiescent-phase entry point is the reader op.
|
||||
const int device_index = static_cast<int>(logits.get_device_index());
|
||||
params.exact_counters = stream_is_capturing(stream)
|
||||
? get_exact_counters(device_index)
|
||||
: ensure_exact_counters(device_index, stream);
|
||||
// Note: this check is UNCONDITIONAL -- eager as well as
|
||||
// capturing, and in both arms. Previously only a capturing launch was
|
||||
// rejected, so an eager launch ran with nullptr and the reader then
|
||||
// reported [0,0,0,0,1]: an unavailable instrument encoded as healthy zero
|
||||
// counts, which is the exact failure the counters exist to prevent.
|
||||
// MERLIN_TOPK_EXACT_MEDIUM=0 is NOT an escape: the control arm's whole job
|
||||
// is to report the overflow incidence it is the control for, so it needs
|
||||
// the counters just as much.
|
||||
STD_TORCH_CHECK(
|
||||
params.exact_counters != nullptr,
|
||||
"persistent_topk: the medium-path exact-restart counters are unavailable on "
|
||||
"device ", device_index,
|
||||
" (not initialised before cudagraph capture, or the one-time "
|
||||
"allocation failed). Call "
|
||||
"torch.ops._C.persistent_topk_exact_counters(out, 0, 1) once per "
|
||||
"device in a quiescent phase before capture.");
|
||||
|
||||
#define LAUNCH_PERSISTENT(TOPK_VAL, VS) \
|
||||
do { \
|
||||
@@ -294,3 +779,249 @@ void persistent_topk(const torch::stable::Tensor& logits,
|
||||
STD_TORCH_CHECK(false, "persistent_topk is not supported on ROCm");
|
||||
#endif
|
||||
}
|
||||
|
||||
// Read (and, on first call, allocate) the medium-path exact-restart
|
||||
// counters. This is also the explicit pre-capture init path: call it once per
|
||||
// device before any cudagraph capture.
|
||||
//
|
||||
// PROTOCOL VERSION 2 -- BREAKING. `out` is an **int64**
|
||||
// CUDA tensor with EXACTLY kNumTopKExactCounters (11) elements; int32, and any
|
||||
// other length, are REJECTED rather than silently accepted. The device counters are 64-bit unsigned
|
||||
// and are copied D2D without conversion, so the reader's element width must
|
||||
// match the device's.
|
||||
//
|
||||
// Why the break rather than a reinterpreting reader: the counters were u32
|
||||
// copied into a SIGNED int32 tensor. At 65,536 columns with k=512 an
|
||||
// all-equal row drops 65,024 ties, so [6] goes negative in Python after 33,027
|
||||
// rows and wraps modulo 2^32 after 66,053 -- and a subtraction across the
|
||||
// signed boundary gives a wrong delta, which is worse than a missing number
|
||||
// because it looks like a measurement. Keeping u32 and reinterpreting in the
|
||||
// reader would need an enforced per-window increment bound plus quiescent
|
||||
// resets, i.e. a contract every future caller has to remember. 64-bit device
|
||||
// counters make the failure unreachable instead: `atomicAdd(unsigned long
|
||||
// long*)` is native on sm_86 (it is a single ATOM.E.ADD.S64, not a CAS loop),
|
||||
// so the instrument costs the same, and a 64-bit slot cannot be driven to the
|
||||
// signed boundary by any run this kernel will ever see.
|
||||
//
|
||||
// The layout is the UNION of the tie-determinism and large-path slot
|
||||
// maps -- the large path's ten slots plus the decode restart's [10] decode_overflow, eleven in all,
|
||||
// with nothing renumbered. The two switch flags are slots INSIDE the array at
|
||||
// [8] and [9], not two elements appended after it, so a probe sized to the
|
||||
// old ten- or five-element layout is rejected by the length check below rather
|
||||
// than being filled with the wrong meanings. Ask the build for its own answer
|
||||
// with `torch.ops._C.persistent_topk_exact_counters_slots()`.
|
||||
//
|
||||
// Everything that reads these counters must move together: the Python reader
|
||||
// `vllm/models/qwen4_exp/nvidia/ops/topk_counters.py` allocates int64 and
|
||||
// eleven slots, and the medium- and large-path tests do too. An int32 caller gets a loud
|
||||
// error rather than half a number. The OPERATOR SCHEMA is unchanged (dtype
|
||||
// and length are not part of it), so a stale caller fails at the check, not at
|
||||
// the binding.
|
||||
//
|
||||
// Slots:
|
||||
// [0] medium-path rows entered (denominator)
|
||||
// [1] rows where a 4,096-slot guard would drop a candidate
|
||||
// [2] rows that took the exact restart (0 in the control arm)
|
||||
// [3] exact restarts that could not complete (must stay 0)
|
||||
// [4] large-path rows emitted deterministically (0 in the control arm)
|
||||
// [5] large-path rows whose pivot-equal group was cut at the k-th value
|
||||
// [6] large-path pivot-equal elements discarded by those cuts
|
||||
// [7] large-path invariant violations (must stay 0)
|
||||
// [8] 1 if MERLIN_TOPK_EXACT_MEDIUM is on, else 0 (host-written)
|
||||
// [9] 1 if MERLIN_TOPK_DET_LARGE is on, else 0 (host-written)
|
||||
// [10] decode rows whose buffered candidates exceeded DBUF (3,708)
|
||||
// before deterministic ranking -- MUST be read before any decode-path
|
||||
// determinism claim
|
||||
//
|
||||
// COHERENCY -- stated precisely, because the earlier wording overclaimed.
|
||||
// With `sync == 0` the snapshot is ordered ONLY against work already enqueued
|
||||
// on the current stream; a kernel running on another stream can land
|
||||
// increments it misses. With `sync != 0` the call synchronizes the device
|
||||
// before the copy AND again after the copy/reset have been issued, so on
|
||||
// return every increment submitted before the call is included and any reset
|
||||
// has COMPLETED. That is still conditional on the CALLER HOLDING EXCLUSIVE
|
||||
// SUBMISSION OWNERSHIP: nothing stops another thread enqueueing work after the
|
||||
// leading synchronize, and no claim of unconditional global coherence is made
|
||||
// here. A stronger contract would need an event every later producer waits
|
||||
// on; this op deliberately implements the simpler half and says so.
|
||||
//
|
||||
// `reset != 0` REQUIRES `sync != 0`. Reset between measurements, never during
|
||||
// one.
|
||||
//
|
||||
// Must not be called during cudagraph capture (it allocates, synchronizes and
|
||||
// issues a host-to-device copy). It IS the explicit, quiescent-phase
|
||||
// initialization path: call it once per device before any capture.
|
||||
void persistent_topk_exact_counters(torch::stable::Tensor& out, int64_t reset,
|
||||
int64_t sync) {
|
||||
#ifndef USE_ROCM
|
||||
STD_TORCH_CHECK(out.is_cuda(), "out must be a CUDA tensor");
|
||||
STD_TORCH_CHECK(
|
||||
out.scalar_type() == torch::headeronly::ScalarType::Long,
|
||||
"out must be int64: the exact-restart counters are 64-bit as of the "
|
||||
"large-path counter protocol change, and an int32 buffer would be filled "
|
||||
"from half of each counter. Allocate "
|
||||
"torch.zeros(persistent_topk_exact_counters_slots(), dtype=torch.int64)."
|
||||
" (int32 was protocol version 1 and is deliberately rejected.)");
|
||||
// Element count alone is not enough. A CUDA scalar expanded
|
||||
// to ten elements (stride 0) satisfies numel but owns one element's storage,
|
||||
// and the raw int64 writes below would run past it; an ordinary strided view
|
||||
// would receive the values in the wrong slots. Require a contiguous 1-D
|
||||
// buffer before taking the pointer.
|
||||
STD_TORCH_CHECK(out.dim() == 1, "out must be 1-D");
|
||||
STD_TORCH_CHECK(out.is_contiguous(),
|
||||
"out must be contiguous (expanded or strided views would be "
|
||||
"written out of bounds or into the wrong elements)");
|
||||
// EXACTLY the build's slot count, not "at least". `>=` accepted an
|
||||
// 11-element buffer on the ten-slot large-path build and quietly left the extra
|
||||
// element reading 0 -- a caller compiled against one layout got a plausible
|
||||
// answer from a build with another, which is the failure mode the
|
||||
// self-describing `..._slots` op exists to prevent. Ask the build, size the
|
||||
// buffer to what it says, and a mismatch is an error rather than a wrong
|
||||
// number.
|
||||
STD_TORCH_CHECK(out.numel() == vllm::persistent::kNumTopKExactCounters,
|
||||
"out must hold exactly ",
|
||||
vllm::persistent::kNumTopKExactCounters,
|
||||
" elements (this build's persistent_topk_exact_counters_"
|
||||
"slots(); the switch flags are slots inside the array, not "
|
||||
"appended after it), got ", out.numel());
|
||||
STD_TORCH_CHECK(
|
||||
reset == 0 || sync != 0,
|
||||
"persistent_topk_exact_counters: reset requires sync != 0, otherwise the "
|
||||
"zeroing races in-flight kernel atomics and silently drops events");
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
out.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
STD_TORCH_CHECK(!stream_is_capturing(stream),
|
||||
"persistent_topk_exact_counters must not be called during "
|
||||
"cudagraph capture");
|
||||
constexpr size_t bytes =
|
||||
vllm::persistent::kNumTopKExactCounters * sizeof(unsigned long long);
|
||||
|
||||
int64_t* dst = out.mutable_data_ptr<int64_t>();
|
||||
unsigned long long* src =
|
||||
ensure_exact_counters(static_cast<int>(out.get_device_index()), stream);
|
||||
|
||||
// An unavailable buffer used to be reported as four zeros
|
||||
// plus "enabled", i.e. an instrument failure indistinguishable from "no
|
||||
// medium work and no faults". Fail instead.
|
||||
STD_TORCH_CHECK(src != nullptr,
|
||||
"persistent_topk_exact_counters: the counter buffer could "
|
||||
"not be allocated on this device; the instrument is "
|
||||
"unavailable and its absence must not read as zero counts");
|
||||
|
||||
if (sync != 0) {
|
||||
// Everything that could still be adding to the block must have retired
|
||||
// before the snapshot (and before a reset) for the numbers to mean
|
||||
// anything across streams.
|
||||
STD_TORCH_CHECK(cudaDeviceSynchronize() == cudaSuccess,
|
||||
"exact counter synchronize failed");
|
||||
}
|
||||
|
||||
STD_TORCH_CHECK(
|
||||
cudaMemcpyAsync(dst, src, bytes, cudaMemcpyDeviceToDevice, stream) ==
|
||||
cudaSuccess,
|
||||
"exact counter read failed");
|
||||
if (reset != 0) {
|
||||
STD_TORCH_CHECK(cudaMemsetAsync(src, 0, bytes, stream) == cudaSuccess,
|
||||
"exact counter reset failed");
|
||||
}
|
||||
|
||||
// Static storage: the source of an async H2D copy must outlive the copy.
|
||||
// Written AFTER the device copy, which has already zeroed both slots: they
|
||||
// are slots [8] and [9] INSIDE the array and the device never touches them.
|
||||
// The large flag reports the ENV value, not the per-launch effective value:
|
||||
// a launch whose ctas_per_group overflowed the staging row still reports 1
|
||||
// here and leaves counter [4] at 0, which is how that degradation is
|
||||
// diagnosed. kSlotSwitchLarge == kSlotSwitchMedium + 1, so one copy does
|
||||
// both.
|
||||
static_assert(vllm::persistent::kSlotSwitchLarge ==
|
||||
vllm::persistent::kSlotSwitchMedium + 1,
|
||||
"the two host-written flags must stay adjacent for this copy");
|
||||
static const int64_t enabled[2] = {exact_medium_enabled() ? 1 : 0,
|
||||
det_large_enabled() ? 1 : 0};
|
||||
STD_TORCH_CHECK(
|
||||
cudaMemcpyAsync(dst + vllm::persistent::kSlotSwitchMedium, enabled,
|
||||
2 * sizeof(int64_t), cudaMemcpyHostToDevice,
|
||||
stream) == cudaSuccess,
|
||||
"exact counter flag write failed");
|
||||
|
||||
if (sync != 0) {
|
||||
// The leading synchronize alone left the copy and the reset
|
||||
// merely ENQUEUED, so the caller could return and immediately race the
|
||||
// outstanding reset from another stream. Complete the handoff before
|
||||
// returning.
|
||||
STD_TORCH_CHECK(cudaDeviceSynchronize() == cudaSuccess,
|
||||
"exact counter completion synchronize failed");
|
||||
}
|
||||
#else
|
||||
STD_TORCH_CHECK(false, "persistent_topk is not supported on ROCm");
|
||||
#endif
|
||||
}
|
||||
|
||||
// How many int32 slots THIS build's counter array carries.
|
||||
//
|
||||
// The Python reader used to guess from MERLIN_TOPK_COUNTER_SLOTS (default 5)
|
||||
// and name the slots by that length. The decode restart makes the array 11 and the large path makes
|
||||
// it 10, so a guess is now wrong on at least one branch, and a reader that
|
||||
// allocates the wrong length is rejected by the length check above -- the
|
||||
// instrument goes dark exactly when it starts mattering. Ask the build
|
||||
// instead. No CUDA call, no allocation, safe from anywhere.
|
||||
int64_t persistent_topk_exact_counters_slots() {
|
||||
#ifndef USE_ROCM
|
||||
return static_cast<int64_t>(vllm::persistent::kNumTopKExactCounters);
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
// The persistent_topk
|
||||
// switches AS THIS PROCESS CACHED THEM -- the values every launch of this
|
||||
// process uses -- read from the mirrors above. Tensor-free, no CUDA call, no
|
||||
// allocation, never initialises a cache (see publish_cached), so it is safe
|
||||
// from anywhere and asking changes nothing.
|
||||
//
|
||||
// Packed into one int64 so the op needs no tensor and no dispatch key.
|
||||
// 2 bits per field, field i at bits [2i, 2i+1]: 0 = off, 1 = on,
|
||||
// 2 = the static has not been initialised in this process yet.
|
||||
// field 0 MERLIN_TIE_DET (det_ties_enabled)
|
||||
// field 1 MERLIN_TIE_RECENT as parsed (tie_recent_enabled)
|
||||
// field 2 effective direction, 1 = most-recent (effective_tie_recent)
|
||||
// field 3 MERLIN_TOPK_EXACT_MEDIUM (exact_medium_enabled)
|
||||
// field 4 MERLIN_TOPK_EXACT_DECODE (exact_decode_enabled)
|
||||
// field 5 MERLIN_TOPK_DET_LARGE (det_large_enabled)
|
||||
// field 6 MERLIN_TOPK_SORTED_EMIT (sorted_emit_enabled)
|
||||
// bits [48, 55] = field count (7), bits [56, 62] = layout version (2). A
|
||||
// reader must check both and refuse any other layout rather than guess.
|
||||
// Layout 2 = layout 1 plus field 6; fields 0-5 are unchanged, so
|
||||
// a reader that knows both can decode either. Field 6 is what
|
||||
// qsa_indexer.py reads to decide whether its Python sort may be skipped.
|
||||
int64_t persistent_topk_toggle_state() {
|
||||
#ifndef USE_ROCM
|
||||
const auto field = [](int cached) -> int64_t {
|
||||
return cached < 0 ? 2 : (cached != 0 ? 1 : 0);
|
||||
};
|
||||
const int det = g_cached_tie_det.load(std::memory_order_acquire);
|
||||
const int recent = g_cached_tie_recent.load(std::memory_order_acquire);
|
||||
// The kernel's own rule, from the kernel's own header, on the cached values.
|
||||
const int effective =
|
||||
(det < 0 || recent < 0)
|
||||
? -1
|
||||
: (vllm::persistent::effective_tie_recent(det != 0, recent != 0)
|
||||
? 1
|
||||
: 0);
|
||||
int64_t state = 0;
|
||||
state |= field(det) << 0;
|
||||
state |= field(recent) << 2;
|
||||
state |= field(effective) << 4;
|
||||
state |= field(g_cached_exact_medium.load(std::memory_order_acquire)) << 6;
|
||||
state |= field(g_cached_exact_decode.load(std::memory_order_acquire)) << 8;
|
||||
state |= field(g_cached_det_large.load(std::memory_order_acquire)) << 10;
|
||||
state |= field(g_cached_sorted_emit.load(std::memory_order_acquire)) << 12;
|
||||
state |= int64_t{7} << 48;
|
||||
state |= int64_t{2} << 56;
|
||||
return state;
|
||||
#else
|
||||
return 0; // layout version 0: no persistent_topk on ROCm
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -586,6 +586,13 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"persistent_topk(Tensor logits, Tensor lengths, Tensor! output, "
|
||||
"Tensor workspace, int k, int max_seq_len) -> ()");
|
||||
|
||||
// Read (and optionally reset) the medium-path exact-restart counters.
|
||||
ops.def(
|
||||
"persistent_topk_exact_counters(Tensor! out, int reset, int sync) -> ()");
|
||||
ops.def("persistent_topk_exact_counters_slots() -> int");
|
||||
// v2.2 -- the switches as this process cached them (topk.cu for the layout).
|
||||
ops.def("persistent_topk_toggle_state() -> int");
|
||||
|
||||
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
||||
ops.def(
|
||||
"cooperative_topk(Tensor logits, Tensor lengths, Tensor! output, "
|
||||
@@ -858,6 +865,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("top_k_per_row_prefill", TORCH_BOX(&top_k_per_row_prefill));
|
||||
ops.impl("top_k_per_row_decode", TORCH_BOX(&top_k_per_row_decode));
|
||||
ops.impl("persistent_topk", TORCH_BOX(&persistent_topk));
|
||||
ops.impl("persistent_topk_exact_counters",
|
||||
TORCH_BOX(&persistent_topk_exact_counters));
|
||||
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
||||
ops.impl("cooperative_topk", TORCH_BOX(&cooperative_topk));
|
||||
#endif
|
||||
@@ -934,6 +943,15 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CompositeExplicitAutograd, ops) {
|
||||
ops.impl("cutlass_scaled_mm_supports_fp4",
|
||||
TORCH_BOX(&cutlass_scaled_mm_supports_fp4));
|
||||
#endif
|
||||
// Tensor-free: the counter layout the BUILD carries. Registered here, not
|
||||
// in the CUDA block -- with no tensor argument there is no dispatch key, so
|
||||
// a CUDA-only registration can never be called (reference host:
|
||||
// NotImplementedError "no fallback function is registered").
|
||||
ops.impl("persistent_topk_exact_counters_slots",
|
||||
TORCH_BOX(&persistent_topk_exact_counters_slots));
|
||||
// Tensor-free for the same reason as the slot count above.
|
||||
ops.impl("persistent_topk_toggle_state",
|
||||
TORCH_BOX(&persistent_topk_toggle_state));
|
||||
}
|
||||
|
||||
// Cache ops
|
||||
|
||||
@@ -132,6 +132,7 @@ markers = [
|
||||
"cpu_test: mark test as CPU-only test",
|
||||
"split: run this test as part of a split",
|
||||
"distributed: run this test only in distributed GPU tests",
|
||||
"gpu: requires a CUDA device; auto-skipped when none is present, and deselectable as a tier with -m 'not gpu'",
|
||||
"optional: optional tests that are automatically skipped, include --optional to run them",
|
||||
]
|
||||
|
||||
|
||||
@@ -212,7 +212,7 @@ class CacheConfig:
|
||||
mamba_block_size is a multiple of replayssm_buffer_len, but this is not
|
||||
required."""
|
||||
use_recoverssm: bool = field(default=False, init=False)
|
||||
"""Whether Kimi-K3 KDA uses RecoverSSM speculative decode."""
|
||||
"""Whether RecoverSSM speculative decode is used (Kimi-K3 KDA; Qwen4Exp GDN and PLE)."""
|
||||
|
||||
# Will be set after profiling.
|
||||
num_gpu_blocks: int | None = field(default=None, init=False)
|
||||
|
||||
@@ -767,9 +767,20 @@ class CompilationConfig:
|
||||
"vllm::mamba_mixer2",
|
||||
"vllm::mamba_mixer",
|
||||
"vllm::short_conv",
|
||||
# Qwen4Exp's AMD backend still uses these splitting ops.
|
||||
# Qwen4Exp: the AMD backend and (fork, compile-profile guard) the compile-restored
|
||||
# NVIDIA backend. #55272 dropped the NVIDIA-only ngram-ids split; the
|
||||
# prefetch pair is the UVA PLE side-stream fork/join (#54371).
|
||||
"vllm::qwen4_exp_compute_ple_ngram_ids",
|
||||
"vllm::qwen4_exp_ple_start_prefetch",
|
||||
"vllm::qwen4_exp_ple_finalize_prefetch",
|
||||
"vllm::qwen4_exp_ple_short_conv",
|
||||
"vllm::qwen4_exp_qsa_with_output",
|
||||
# Fork (pull transport with the compile-profile guard, found on the reference host): the PLE pull transport's wait+DMA
|
||||
# mutates a module-owned buffer; inside an Inductor graph it failed
|
||||
# post-grad with "auto_functionalized was not removed" on the stage-0
|
||||
# ranks. As a split point it runs between pieces, like attention.
|
||||
"vllm::ple_offload_wait_copy",
|
||||
"vllm::ple_offload_wait",
|
||||
"vllm::linear_attention",
|
||||
"vllm::qwen_gdn_attention_core",
|
||||
"vllm::qwen_gdn_attention_core_fused_norm_packed",
|
||||
@@ -1379,6 +1390,7 @@ class CompilationConfig:
|
||||
max_num_reqs: int | None = None,
|
||||
is_profiling: bool = False,
|
||||
piecewise_capture_available: bool = True,
|
||||
use_replayssm: bool = False,
|
||||
) -> CUDAGraphMode:
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
|
||||
@@ -1430,7 +1442,11 @@ class CompilationConfig:
|
||||
self.splitting_ops_contain_attention()
|
||||
or self.use_inductor_graph_partition
|
||||
)
|
||||
) or envs.VLLM_USE_BREAKABLE_CUDAGRAPH:
|
||||
) or (
|
||||
envs.VLLM_USE_BREAKABLE_CUDAGRAPH
|
||||
# CompilationConfig has no cache_config; the runner passes the flag.
|
||||
and use_replayssm
|
||||
):
|
||||
# Breakable CUDA graphs break at the attention kernels at runtime,
|
||||
# with or without torch.compile.
|
||||
msg += (
|
||||
|
||||
+19
-3
@@ -3,9 +3,10 @@
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from pydantic import model_validator
|
||||
from pydantic import Field, model_validator
|
||||
from typing_extensions import Self
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.utils import config, get_hash_factors, hash_factors
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -21,6 +22,17 @@ _NGRAM_LAYER_FIELDS = {
|
||||
}
|
||||
|
||||
|
||||
def _default_cpu_offload() -> bool:
|
||||
# Fork: with VLLM_E2_PLE_PULL_TRANSPORT=1 the default is False and a
|
||||
# VLLM_PLE_CPU_OFFLOAD value that would enable UVA fails the boot. With the
|
||||
# pull transport off, the upstream default below is returned unchanged.
|
||||
from vllm.v1.e2_guards import pull_transport_cpu_offload_default
|
||||
|
||||
if not envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
return envs.VLLM_PLE_CPU_OFFLOAD
|
||||
return pull_transport_cpu_offload_default(False)
|
||||
|
||||
|
||||
def model_has_engram_layers(model_config: "ModelConfig | None") -> bool:
|
||||
"""Whether the model carries n-gram embedding layers."""
|
||||
if model_config is None:
|
||||
@@ -35,8 +47,12 @@ def model_has_engram_layers(model_config: "ModelConfig | None") -> bool:
|
||||
class EngramConfig:
|
||||
"""Configuration for Engram embedding storage and sharding."""
|
||||
|
||||
cpu_offload: bool = True
|
||||
"""Store embedding weights in pinned CPU memory for UVA lookup."""
|
||||
cpu_offload: bool = Field(default_factory=_default_cpu_offload)
|
||||
"""Store embedding weights in pinned CPU memory for UVA lookup.
|
||||
Defaults to VLLM_PLE_CPU_OFFLOAD, which is enabled by default. An explicit
|
||||
value takes precedence over the environment variable. With
|
||||
VLLM_E2_PLE_PULL_TRANSPORT=1 the default is False and a value that would
|
||||
enable UVA fails the boot (vllm/v1/e2_guards.py)."""
|
||||
|
||||
host_file_gather: bool = False
|
||||
"""Serve Qwen4Exp PLE rows by gathering them on the CPU from the loader's
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.config.fault_tolerance import FaultToleranceConfig
|
||||
from vllm.config.utils import config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_ports_list
|
||||
from vllm.utils.network_utils import get_open_ports_list, get_open_zmq_ipc_path
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ray.runtime_env import RuntimeEnv
|
||||
@@ -369,6 +369,10 @@ class ParallelConfig:
|
||||
"""Port of the coordination TCPStore. Can be set by the API server; workers
|
||||
connect as clients to exchange self-picked group ports at runtime."""
|
||||
|
||||
_ple_offload_ipc_path: str = ""
|
||||
"""Node-local ZMQ IPC address for the fork's PLE pull-transport offload
|
||||
process. Set only when VLLM_E2_PLE_PULL_TRANSPORT=1."""
|
||||
|
||||
decode_context_parallel_size: int = Field(default=1, ge=1)
|
||||
"""Number of ranks that shard the decode KV cache. DCP does not expand
|
||||
the process world size. Without PCP, DCP reuses TP ranks. With PCP, DCP
|
||||
@@ -517,6 +521,9 @@ class ParallelConfig:
|
||||
"The FT system assumes one AsyncMPClient manages all engines."
|
||||
)
|
||||
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT and not self._ple_offload_ipc_path:
|
||||
self._ple_offload_ipc_path = get_open_zmq_ipc_path()
|
||||
|
||||
if self.all2all_backend in ["pplx", "naive"]:
|
||||
logger.warning(
|
||||
"The '%s' all2all backend has been removed. "
|
||||
@@ -910,6 +917,7 @@ class ParallelConfig:
|
||||
"worker_extension_cls",
|
||||
"_api_process_count",
|
||||
"_api_process_rank",
|
||||
"_ple_offload_ipc_path",
|
||||
# NUMA binding is per-rank host-side memory locality; it does
|
||||
# not affect collective-communication semantics. When numa_bind
|
||||
# is enabled with auto-detection, each DP rank stores its own
|
||||
|
||||
+40
-10
@@ -106,9 +106,12 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
|
||||
"KimiLinearForCausalLM",
|
||||
"MiniMaxM3SparseForCausalLM",
|
||||
"MiniMaxM3SparseForConditionalGeneration",
|
||||
"Qwen4ExpForCausalLM",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
"Qwen4ExpMTP",
|
||||
# Fork (compile-profile guard): Qwen4ExpForCausalLM,
|
||||
# Qwen4ExpForConditionalGeneration and Qwen4ExpMTP are deliberately
|
||||
# NOT here. #55272 added them when it removed torch.compile from the
|
||||
# NVIDIA path; this tree restores compile, so the resolved profile for
|
||||
# them is VLLM_COMPILE + FULL_AND_PIECEWISE with breakable capture off
|
||||
# (vllm/v1/e2_guards.py asserts it at model construction).
|
||||
}
|
||||
)
|
||||
|
||||
@@ -1249,6 +1252,11 @@ class VllmConfig:
|
||||
# PyTorch's expandable_segments allocator uses CUDA VMM, which can
|
||||
# remap a virtual address range to different physical pages over the
|
||||
# engine's lifetime. KV connectors that pin KV cache memory (e.g.
|
||||
# Fork: an explicit cpu_offload=True beside the pull transport is
|
||||
# a double offload; refused here, before any worker starts.
|
||||
from vllm.v1.e2_guards import verify_engram_against_pull
|
||||
|
||||
verify_engram_against_pull(self.engram_config)
|
||||
# NixlConnector via ibv_reg_mr, MooncakeConnector) end up with their
|
||||
# registrations pointing at stale physical pages after any remap,
|
||||
# producing RDMA failures like IBV_WC_REM_ACCESS_ERR /
|
||||
@@ -3522,11 +3530,17 @@ class VllmConfig:
|
||||
and not self.cache_config.use_recoverssm
|
||||
):
|
||||
# Qwen4Exp's GDN and PLE layers have no plain ReplaySSM path; with them the
|
||||
# flag selects RecoverSSM, which needs speculative decoding.
|
||||
raise ValueError(
|
||||
"--use-replayssm on Qwen4Exp selects GDN RecoverSSM and requires "
|
||||
"speculative decoding (num_speculative_tokens > 0)"
|
||||
# flag selects RecoverSSM, which needs speculative decoding. Fork (v2.5):
|
||||
# without speculative decoding there are no drafts to recover from, so the
|
||||
# flag falls back to the stock path (as if it were not passed) instead of
|
||||
# refusing to start; one launch line then serves with and without MTP.
|
||||
logger.warning(
|
||||
"--use-replayssm on Qwen4Exp selects GDN RecoverSSM, which needs "
|
||||
"speculative decoding; num_speculative_tokens=0, so the stock GDN "
|
||||
"path is used (as without --use-replayssm)"
|
||||
)
|
||||
self.cache_config.use_replayssm = False
|
||||
return self
|
||||
if (
|
||||
self.mamba_config.backend == MambaBackendEnum.FLASHINFER
|
||||
and self.cache_config.replayssm_buffer_len > 16
|
||||
@@ -3557,9 +3571,25 @@ class VllmConfig:
|
||||
"RecoverSSM with align mode requires VLLM_USE_V2_MODEL_RUNNER=1"
|
||||
)
|
||||
if self.parallel_config.pipeline_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"RecoverSSM currently requires pipeline_parallel_size=1"
|
||||
)
|
||||
# Non-last PP ranks get the accepted counts pp_size steps after
|
||||
# their verify. Model Runner V2 defers the commit with a
|
||||
# snapshot of its inputs (RecoverSSMState.detach_step) and
|
||||
# masks freed rows; only the Qwen4Exp GDN/PLE metadata
|
||||
# implement that snapshot (Kimi-K3 KDA does not).
|
||||
if (
|
||||
self.model_config is None
|
||||
or self.model_config.architecture
|
||||
not in _QWEN4EXP_RECOVERSSM_ARCHS
|
||||
):
|
||||
raise ValueError(
|
||||
"RecoverSSM with pipeline_parallel_size > 1 is supported "
|
||||
"only for Qwen4Exp GDN/PLE"
|
||||
)
|
||||
if not self.use_v2_model_runner:
|
||||
raise ValueError(
|
||||
"RecoverSSM with pipeline_parallel_size > 1 requires "
|
||||
"VLLM_USE_V2_MODEL_RUNNER=1"
|
||||
)
|
||||
if self.mamba_config.backend != MambaBackendEnum.TRITON:
|
||||
raise ValueError("RecoverSSM requires --mamba-backend triton")
|
||||
elif use_mamba_replayssm_spec:
|
||||
|
||||
@@ -182,6 +182,22 @@ os.environ.setdefault("TRITON_CACHE_AUTOTUNING", "1")
|
||||
# debug layout (see https://github.com/vllm-project/vllm/issues/41410).
|
||||
os.environ.setdefault("TILELANG_CLEANUP_TEMP_FILES", "1")
|
||||
|
||||
# Fork (v2.5, knob-absent boot): the MERLIN_* kernel toggles are read by
|
||||
# getenv in C++ (FULL_K, TIE_DET, TIE_RECENT, TOPK_SORTED_EMIT) and by
|
||||
# vllm.envs (QSA_SORT) with OFF as the code default, while every shipped
|
||||
# launcher sets them to 1. Unset now means the shipped value; an explicit "0"
|
||||
# still turns a toggle off. Set here because this module runs on `import vllm`,
|
||||
# before any reader, and worker processes inherit the environment.
|
||||
_SHIPPED_KERNEL_TOGGLE_DEFAULTS = (
|
||||
"MERLIN_FULL_K",
|
||||
"MERLIN_QSA_SORT",
|
||||
"MERLIN_TIE_DET",
|
||||
"MERLIN_TIE_RECENT",
|
||||
"MERLIN_TOPK_SORTED_EMIT",
|
||||
)
|
||||
for _name in _SHIPPED_KERNEL_TOGGLE_DEFAULTS:
|
||||
os.environ.setdefault(_name, "1")
|
||||
|
||||
# ===================================================
|
||||
# torch 2.9 Inductor PythonWrapperCodegen monkeypatch
|
||||
# ===================================================
|
||||
|
||||
+583
@@ -304,6 +304,195 @@ if TYPE_CHECKING:
|
||||
VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024
|
||||
VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary"
|
||||
VLLM_USE_V2_MODEL_RUNNER: bool | None = None
|
||||
# Compile-cache factor (v2 host-embedding work): the pinned-host UVA embedding
|
||||
# table changes the traced MTP-drafter graph (reads _uva_weight instead of
|
||||
# embed_tokens.weight). Registered here so envs.compile_factors() keys the
|
||||
# AOT cache on it; the layer itself still reads os.environ directly.
|
||||
VLLM_HOST_EMBED_TABLE: bool = False
|
||||
VLLM_PLE_OFFLOAD_READY_TIMEOUT: float = 600.0
|
||||
# --- Flash-Next fork knobs (v2.2) --------------------------------
|
||||
# Every name below was read straight from os.environ at its use site, so
|
||||
# envs.validate_environ() logged it as unknown on every boot that set one
|
||||
# and compile_factors() could not see it. Registering makes the hash
|
||||
# complete; the read sites are unchanged (they still parse and fail closed
|
||||
# themselves, which registration must not weaken). Runtime-only entries are
|
||||
# listed in compile_factors().ignored_factors with their reason.
|
||||
#
|
||||
# TWO IDENTITIES, NOT ONE. "Read once at startup"
|
||||
# settles neither of them, so every entry below is labelled with both:
|
||||
#
|
||||
# AOT identity -- does the value change the torch.compile / AOT
|
||||
# artifact? This is the ONLY identity compile_factors()
|
||||
# keys. It protects against reusing another process's
|
||||
# cached artifact.
|
||||
# CAPTURE identity -- does the value change what a cudagraph capture bakes
|
||||
# in (ops present, launch constants, shapes)? Nothing
|
||||
# in this tree keys anything on capture identity: the
|
||||
# env is read before capture, in-process, so capture
|
||||
# identity is protected only by "do not change the
|
||||
# value between capture and replay in one process".
|
||||
#
|
||||
# So a knob that is a capture factor but NOT an AOT factor gets no
|
||||
# protection from being registered here. Registering it is a contract
|
||||
# statement and a cross-process guard, not a mechanism. Where the two
|
||||
# identities are not both established by reading the code, the knob is
|
||||
# hashed anyway: a spurious cache miss costs a recompile, a missed one costs
|
||||
# a wrong graph.
|
||||
#
|
||||
# *** INERT ON THE 0.30 BASE. ***
|
||||
# NO READ SITE, and therefore NOT HASHED -- moved out of the compile
|
||||
# factors into ignored_factors below. Its only reader was the 21-line
|
||||
# catch-all branch in _get_ple_embedding_quant_method, which lived in the
|
||||
# half of models/qwen4_exp/nvidia/ple_layer.py that #54371 deleted. With no
|
||||
# reader it cannot change an artifact, so hashing it would only invalidate
|
||||
# caches for nothing. (The AOT artifact is real on this tree: #55272
|
||||
# removed torch.compile from the NVIDIA Qwen4Exp path, the compile-profile guard restored it,
|
||||
# and the vLLM 0.30 rebase ran AOT cache-load boots -- so the reason is "no
|
||||
# reader", not "no artifact".) It gives NO capture protection either.
|
||||
# v0.30.0 selects the FP8 PLE method from the CONFIG KEY
|
||||
# text_config.ple_embedding_dtype == "float8_e4m3fn"
|
||||
# (ngram_embedding.py:174-175) -- see
|
||||
# tests/models/qwen4_exp/test_ple_fp8_selector_checkpoint.py.
|
||||
# Setting this on the 0.30 base does NOTHING, and NOTHING WARNS: validate_environ
|
||||
# (envs.py, below) warns only about names that are NOT registered, and this
|
||||
# comment lives in a TYPE_CHECKING block that never runs. It is kept
|
||||
# registered so a later PLE transport change can reland its reader and
|
||||
# reclassify it, or delete it. DO NOT assume it works.
|
||||
# When its reader returns: AOT yes, CAPTURE yes -- the FP8-with-global-scale
|
||||
# table selects Qwen4ExpPLEFp8EmbeddingMethod over the checkpoint's method,
|
||||
# a different weight dtype and a different embedding forward.
|
||||
VLLM_PLE_FP8_GLOBAL_SCALE: bool = False
|
||||
#
|
||||
# COMPILE/CAPTURE FACTORS (hashed):
|
||||
# CAPTURE: yes, established -- clip counting allocates persistent per-layer
|
||||
# int64 counters and its increment ops are CAPTURED into the cudagraph (see
|
||||
# the graph-safety note in models/qwen4_exp/nvidia/qsa.py); unset, those ops
|
||||
# are absent. AOT: not established (the QSA forward is a custom op); hashed
|
||||
# conservatively.
|
||||
VLLM_QSA_KV_CLIP_COUNT: str = ""
|
||||
# CAPTURE: n/a -- collection requires --enforce-eager, so an enabled run
|
||||
# does not capture at all. AOT: adds absmax reduction ops to the attention
|
||||
# forward; hashed conservatively.
|
||||
VLLM_QSA_KV_COLLECT: str = ""
|
||||
# CAPTURE: yes -- the resolved value is a launch constant baked into the
|
||||
# graph at capture (csrc/libtorch_stable/persistent_topk.cuh:143-148).
|
||||
# AOT: not established. Hashed conservatively.
|
||||
# NOT A PYTHON READ SITE: the value is resolved once per process by
|
||||
# exact_medium_enabled() in csrc/libtorch_stable/topk.cu:30-36, which is the
|
||||
# single source of truth. This entry is a raw passthrough so nothing can
|
||||
# mistake it for a second parser; do not use envs.MERLIN_TOPK_EXACT_MEDIUM
|
||||
# to decide behaviour. Registered with the medium-path exact restart
|
||||
# so the cache-factor contract is complete.
|
||||
MERLIN_TOPK_EXACT_MEDIUM: str | None = None
|
||||
# The fork's PLE pull transport -- the host-mapped,
|
||||
# fail-closed offload process in vllm/v1/ple_offload/. Renamed from the E2
|
||||
# base's VLLM_PLE_CPU_OFFLOAD, which vLLM v0.30.0 reuses for a DIFFERENT
|
||||
# subsystem with the OPPOSITE default (upstream: pinned-host UVA lookup,
|
||||
# default ON). The two subsystems no longer share a switch.
|
||||
# Strict: unset or "0" = off, "1" = on, anything else is a boot error.
|
||||
# On: the PLE table lives in the offload process, the GPU placeholder waits
|
||||
# on a host flag and DMAs rows inside the captured forward, and upstream's
|
||||
# UVA backend is refused (vllm/v1/e2_guards.py). Off: the tree is upstream.
|
||||
# AOT: yes (different module class and forward). CAPTURE: yes (the captured
|
||||
# graph contains the wait/copy ops instead of a table lookup). Hashed.
|
||||
# UNSET = ON (since v2.5), the shipped value: upstream's UVA backend (the off
|
||||
# path) compiles an fp8e4nv Triton kernel that sm_86 cannot build, so a launcher
|
||||
# without this switch failed at profile_run. "0" still selects upstream's path.
|
||||
VLLM_E2_PLE_PULL_TRANSPORT: bool = True
|
||||
#
|
||||
# RUNTIME-ONLY (registered for validate_environ + a single cached read):
|
||||
# Static K/V scale sidecar: fill_()ed into existing fp32 device buffers
|
||||
# after load and before capture, and the kernel does tl.load(k_scale_ptr),
|
||||
# so the value is never a constexpr and the graph is identical.
|
||||
VLLM_QSA_KV_SCALES: str = ""
|
||||
# Guard telemetry: mode raise|warn|off and the reader poll interval in
|
||||
# seconds. counter() returns a real tensor in every mode, so the Triton
|
||||
# arguments and the captured graph do not depend on either.
|
||||
VLLM_E2_GUARD_MODE: str = "warn"
|
||||
VLLM_E2_GUARD_INTERVAL: str = "0.05"
|
||||
# Debug allocator telemetry (poll seconds + output directory) and the
|
||||
# scheduler allocation log path.
|
||||
VLLM_E2_MEMSTATS: str = "0"
|
||||
VLLM_E2_MEMSTATS_DIR: str = ""
|
||||
VLLM_E2_SCHED_LOG: str = ""
|
||||
# O(live blocks) host-side ownership assertion in the scheduler, which is
|
||||
# plain Python and never compiled.
|
||||
VLLM_E2_ZERO_OWNERSHIP_CHECK: bool = False
|
||||
# PLE offload child placement: rewrites CUDA_VISIBLE_DEVICES for the child
|
||||
# only (CUDA_VISIBLE_DEVICES is itself an ignored factor).
|
||||
VLLM_PLE_OFFLOAD_HOME_DEVICE: str = ""
|
||||
# PLE timeouts / deadlines, in seconds.
|
||||
VLLM_PLE_OFFLOAD_WAIT_TIMEOUT: float = 120.0
|
||||
VLLM_PLE_PROGRESS_DEADLINE_S: float = 300.0
|
||||
VLLM_PLE_STARTUP_DEADLINE_S: float = 1800.0
|
||||
# Test-only PLE fault injection: cut spec, marker log, bootstrap gate.
|
||||
# AOT: no. CAPTURE: no. Read once at import by the vllm_ple_test_fault
|
||||
# singleton, but the process that reads them is a real engine, so
|
||||
# validate_environ sees them.
|
||||
VLLM_PLE_TEST_FAULT: str | None = None
|
||||
VLLM_PLE_TEST_FAULT_GATE: str | None = None
|
||||
VLLM_PLE_TEST_FAULT_LOG: str | None = None
|
||||
#
|
||||
# --- Knobs registered ahead of their read sites (v2.2) -----------
|
||||
# Registered ahead of their code so the MERGED tree is complete and
|
||||
# validate_environ is quiet from the first boot (a
|
||||
# cross-change knob table). Each mirrors its read site's
|
||||
# parse; if a read site changes, the entry here must be reconciled.
|
||||
# (Both read sites below are now live.)
|
||||
# LIVE: read once at
|
||||
# import as envs.VLLM_E2_PLE_FLAG_CHECK by vllm/v1/ple_offload/connector.py
|
||||
# (the PLE pull-transport flag check, rank-gated, off by default). On
|
||||
# the 0.30 base it had NO read site: the PLE-diagnostics half
|
||||
# had not been relanded after the PLE pull-transport port restored connector.py. The
|
||||
# census in tests/test_envs.py now asserts the read site exists.
|
||||
# AOT: no. CAPTURE: no -- strictly host-side and outside capture.
|
||||
# off|warn|raise (1 = warn), normalised by _e2_diagnostic_mode like the
|
||||
# rest of the family. The mamba ownership check moved this gate off a plain
|
||||
# boolean; reconciled here.
|
||||
VLLM_E2_PLE_FLAG_CHECK: str = "off"
|
||||
# AOT: no. CAPTURE: no -- strictly host-side. off|warn|raise (1 = warn).
|
||||
# the mamba slot-ownership check, v1/worker/mamba_utils.py:74.
|
||||
VLLM_E2_MAMBA_SLOT_OWNERSHIP: str = "off"
|
||||
# v2.5: keep each running request's newest hashed align-mode Mamba
|
||||
# state (its prefix-cache checkpoint) referenced until the request ends,
|
||||
# instead of returning it to the free queue the moment decode passes it
|
||||
# (single_type_kv_cache_manager.MambaManager._retire_state_blocks). Strict
|
||||
# "0"/"1", default ON; "0" restores the old release-on-retire behaviour
|
||||
# for A/B. AOT: no. CAPTURE: no -- scheduler-side block bookkeeping only;
|
||||
# the worker sees the same block tables either way.
|
||||
VLLM_E2_MAMBA_RETAIN_CHECKPOINT: bool = True
|
||||
# v2.5 side-cache layout (Flash-Next PLE + QSA ring): stop charging each
|
||||
# tiny side cache a whole packed pool block per request. The TP-replicated
|
||||
# PLE short-conv state group is folded into a GDN group that has a spare
|
||||
# page on every PP stage, and the one-block-per-request QSA ring group packs
|
||||
# several requests' rings into one pool block. Strict "0"/"1", default ON;
|
||||
# "0" restores the old one-group-each layout for A/B. Read in
|
||||
# vllm/v1/core/kv_cache_utils.py (get_kv_cache_configs). AOT: yes, hashed --
|
||||
# it changes the ring views' block stride and which block table the PLE
|
||||
# state is read through, so ON and OFF must not share a compile cache.
|
||||
VLLM_E2_SIDE_CACHE_LAYOUT: bool = True
|
||||
# Profiling-teardown leak policy (vllm/v1/worker/gpu/cudagraph_utils.py): a profiling
|
||||
# KV cache still allocated after teardown fails the boot by default; "0", "false" or
|
||||
# "no" downgrade it to a warning while diagnosing. AOT: no. CAPTURE: no -- it only
|
||||
# decides whether a host-side check raises.
|
||||
VLLM_FAIL_ON_PROFILING_KV_LEAK: bool = True
|
||||
# Opt-out for the fork startup guards in vllm/v1/e2_guards.py, read at
|
||||
# Qwen4ExpForCausalLM construction. AOT: no. CAPTURE: no -- it changes
|
||||
# nothing itself; it only permits a boot whose identity IS different
|
||||
# (VLLM_USE_BREAKABLE_CUDAGRAPH, which upstream already registers and
|
||||
# hashes). Ignored factor for exactly that reason.
|
||||
VLLM_E2_ALLOW_BREAKABLE: bool = False
|
||||
# Expected RESOLVED cudagraph mode by name, asserted by the same guard when
|
||||
# set (e.g. "FULL", "FULL_DECODE_ONLY"). AOT: no. CAPTURE: no -- it asserts
|
||||
# an identity, it does not choose one. Ignored factor.
|
||||
VLLM_E2_EXPECTED_CUDAGRAPH_MODE: str = ""
|
||||
#
|
||||
# The unaudited-knob fence that stood here listed the
|
||||
# tie-determinism toggles as not-yet-in-tree. They landed with the
|
||||
# v2.2 kernel port: the four are registered and
|
||||
# classified in the "determinism toggles" block below, and the
|
||||
# one knob still unclassified sits in that block's own fence, which
|
||||
# tests/test_envs.py reads as a set of identifiers.
|
||||
VLLM_LOG_MODEL_INSPECTION: bool = False
|
||||
VLLM_DEBUG_MFU_METRICS: bool = False
|
||||
VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: bool = False
|
||||
@@ -328,6 +517,167 @@ if TYPE_CHECKING:
|
||||
VLLM_GPU_NIC_PCIE_MAPPING: str = ""
|
||||
VLLM_NIC_SELECTION_VARS: str = ""
|
||||
VLLM_ENABLE_HPC_OPS: bool = False
|
||||
MERLIN_TOPK_EXACT_COUNTERS_LOG: int = 0
|
||||
# Pre-tie-determinism fallback for the counter reader's slot count; runtime-only
|
||||
# (ignored factor), never consulted on this build. See the dict entry.
|
||||
MERLIN_TOPK_COUNTER_SLOTS: str | None = None
|
||||
#
|
||||
# --- determinism toggles -----------------------------------------
|
||||
# The three nondeterminism mechanisms (FULL_K, QSA_SORT, TIE_DET/TIE_RECENT).
|
||||
# Every one of them is
|
||||
# OFF by default in this file; vllm/env_override.py sets each to 1 (the shipped value) when unset.
|
||||
#
|
||||
# Why these are registered at all: validate_environ() only inspects names
|
||||
# beginning with VLLM_, so a MERLIN_ name never trips the unknown-variable
|
||||
# warning. Registration here exists for ONE reason -- compile_factors() can
|
||||
# only hash names it knows, and compile_factors() is a deny-list, so
|
||||
# registering a name hashes it unless ignored_factors says otherwise. All
|
||||
# four are deliberately absent from ignored_factors.
|
||||
#
|
||||
# CAPTURE: yes, all four. Each resolves once per process and the resolved
|
||||
# value then reaches the GPU as a kernel launch argument, a launch
|
||||
# configuration, or an extra op in the traced sequence -- so a cudagraph
|
||||
# bakes in whatever was live at capture time. Flipping one between boots
|
||||
# changes the captured graph, which is exactly what a cache key is for.
|
||||
# AOT: not established for the three CUDA toggles (the Marlin and topk
|
||||
# kernels are compiled into the extension at build time, not by the AOT
|
||||
# compiler); established for MERLIN_QSA_SORT, which adds torch ops to the
|
||||
# traced graph. Hashed conservatively throughout, the same rule
|
||||
# already applied to MERLIN_TOPK_EXACT_MEDIUM.
|
||||
#
|
||||
# NOT A PYTHON READ SITE (FULL_K / TIE_DET / TIE_RECENT): each is resolved
|
||||
# once by a std::getenv in C++ -- merlin_full_k() in
|
||||
# csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu and merlin_det_ties() /
|
||||
# merlin_tie_recent() in csrc/libtorch_stable/topk.cu -- and that is the
|
||||
# single source of truth. The entries below are raw passthroughs so nothing
|
||||
# can mistake them for a second parser; do not use them to decide
|
||||
# behaviour. A raw passthrough hashes "0" and "00" differently even though
|
||||
# the C++ treats them alike: over-invalidation, which is the safe direction.
|
||||
#
|
||||
# Marlin MoE split-K reduction order; the CUDA half of unmerged upstream
|
||||
# PR #46639 (`use_full_k`).
|
||||
MERLIN_FULL_K: str | None = None
|
||||
# persistent_topk tie-break at the k-th cutoff, and its direction.
|
||||
MERLIN_TIE_DET: str | None = None
|
||||
MERLIN_TIE_RECENT: str | None = None
|
||||
#
|
||||
# A REAL Python read site, unlike the three above: parsed here and read
|
||||
# once at import by models/qwen4_exp/nvidia/ops/qsa_indexer.py, which adds
|
||||
# a torch.sort + two torch.where ops to the QSA selection path.
|
||||
MERLIN_QSA_SORT: bool = False
|
||||
#
|
||||
# --- persistent_topk kernel switches ------------------------------
|
||||
# Same classification as the three raw passthroughs above and registered
|
||||
# for the same single reason: compile_factors() is a deny-list, so a name
|
||||
# it does not know cannot be hashed, and a Ray deployment that propagates
|
||||
# registered variables can otherwise drop an explicitly-disabled switch
|
||||
# and leave a worker on the default. Each is resolved ONCE per process by
|
||||
# a std::getenv in csrc/libtorch_stable/topk.cu -- that is the single
|
||||
# source of truth -- and the resolved value reaches the GPU as a kernel
|
||||
# launch argument, so a cudagraph bakes in whatever was live at capture.
|
||||
# CAPTURE: yes, all three. AOT: not established (these kernels are built
|
||||
# into the extension, not by the AOT compiler); hashed conservatively
|
||||
# anyway, same rule as above. None is in ignored_factors.
|
||||
#
|
||||
# Raw passthroughs, so "unset" reads as None here. That is NOT the same as
|
||||
# "off" for the C++: MERLIN_TOPK_EXACT_MEDIUM defaults ON. Read the getter
|
||||
# in topk.cu, not this default, for the shipped behaviour.
|
||||
#
|
||||
# Take the exact restart on an overflowing MEDIUM row:
|
||||
# MERLIN_TOPK_EXACT_MEDIUM. Declared ONCE, in the cross-change knob
|
||||
# block above (same raw passthrough, same hashing); the 0.30 tree already
|
||||
# carried that entry, so it is not repeated here.
|
||||
# The same restart on an overflowing DECODE row.
|
||||
MERLIN_TOPK_EXACT_DECODE: str | None = None
|
||||
# Deterministic placement and tie membership on the LARGE path.
|
||||
# LIVE on this tree since the v2.2 kernel port: the
|
||||
# read site is det_large_enabled() in csrc/libtorch_stable/topk.cu, a
|
||||
# std::getenv resolved once per process, and the resolved value reaches
|
||||
# the large-path radix_topk as a launch argument, so a cudagraph bakes it
|
||||
# in. CAPTURE: yes. AOT: not established (built into the extension, not by
|
||||
# the AOT compiler); hashed conservatively like the switches above. A raw
|
||||
# passthrough because the C++ getter is the single source of truth.
|
||||
MERLIN_TOPK_DET_LARGE: str | None = None
|
||||
#
|
||||
# The large-path FAULT INJECTORS. TEST
|
||||
# ONLY, and registered for exactly the same reason as the switches above,
|
||||
# not because anything should set them in production:
|
||||
#
|
||||
# * both are resolved once per process by a std::getenv in topk.cu
|
||||
# (large_fault_inject_cta(), large_fault_inject_emit()) and the
|
||||
# resolved values reach the GPU as kernel launch arguments, so a
|
||||
# cudagraph BAKES THEM IN. An injected build and a clean build differ
|
||||
# in the captured graph while sharing a cache key -- which is the
|
||||
# failure mode: a graph captured under injection returns an all -1 row
|
||||
# for every deterministic large row, and on replay nothing re-reads the
|
||||
# environment to notice the variable has since been unset. FAULT_INJECT
|
||||
# was once "unregistered and capture-baked"; hashing
|
||||
# them is what stops a cached artifact crossing between the arms.
|
||||
# * compile_factors() is a deny-list, so a name it does not know cannot
|
||||
# be hashed at all. Neither is in ignored_factors.
|
||||
#
|
||||
# Raw passthroughs, like the switches: the C++ getters are the single
|
||||
# source of truth (INJECT parses with atoi and treats <0 and unset alike;
|
||||
# KIND compares against the literal "emit"). Do not parse them here.
|
||||
#
|
||||
# Which CTA of the group has its fault predicate forced; unset / negative
|
||||
# is off, and setting it also suppresses __trap().
|
||||
MERLIN_TOPK_LARGE_FAULT_INJECT: str | None = None
|
||||
# Which fault: unset / "pre" forces the pre-emission predicate caught at
|
||||
# X3, "emit" forces the emission-bound slot check caught at X4. Ignored
|
||||
# unless MERLIN_TOPK_LARGE_FAULT_INJECT names a CTA.
|
||||
MERLIN_TOPK_LARGE_FAULT_KIND: str | None = None
|
||||
#
|
||||
# MERLIN_TOPK_SORTED_EMIT -- persistent_topk
|
||||
# emits every row in the canonical order MERLIN_QSA_SORT imposes
|
||||
# (ascending index, padding last as -1), and qsa_indexer.py then skips its
|
||||
# Python sort. Same classification as the kernel switches above: resolved
|
||||
# ONCE per process by a std::getenv in csrc/libtorch_stable/topk.cu
|
||||
# (sorted_emit_enabled) -- the single source of truth -- and the resolved
|
||||
# value reaches the GPU as a kernel launch argument AND removes three torch
|
||||
# ops from the captured sequence, so a cudagraph bakes it in. CAPTURE: yes.
|
||||
# AOT: not established for the kernel half; hashed conservatively (a fork
|
||||
# env knob must be a compile factor: an os.environ knob that is not
|
||||
# registered is not in the cache key). Raw passthrough: Python never parses
|
||||
# it to decide anything -- qsa_indexer.py reads the C++ cache through
|
||||
# persistent_topk_toggle_state() field 6. Not in ignored_factors.
|
||||
MERLIN_TOPK_SORTED_EMIT: str | None = None
|
||||
# Debug companion, a REAL Python read site (qsa_indexer._SORTED_EMIT_CHECK,
|
||||
# read once at import): when the kernel sorted, assert on device that the
|
||||
# row equals its canonical form instead of trusting it. Adds ops to the
|
||||
# captured sequence, so it is hashed too. Default off.
|
||||
MERLIN_TOPK_SORTED_EMIT_CHECK: bool = False
|
||||
#
|
||||
# NOT REGISTERED, deliberately: MERLIN_TOPK_EXACT_TRAP. It is a
|
||||
# preprocessor define consumed by `#ifndef` in persistent_topk.cuh at
|
||||
# BUILD time (-D), not a process environment variable -- nothing in the
|
||||
# tree calls getenv on it. Registering it would advertise a runtime knob
|
||||
# that does not exist. Its default (trap ON) is an open maintainer decision,
|
||||
# not something to settle by
|
||||
# quietly making it settable.
|
||||
#
|
||||
# --- UNAUDITED: reference-host knobs with no read site in this tree ----------
|
||||
# A knob named inside the fence below is one this repository cannot
|
||||
# classify, because the code that reads it is not in it. Registering one
|
||||
# without first deleting it from the fence -- which means writing down its
|
||||
# AOT and capture identity against its real read site -- is the exact
|
||||
# failure the fork's env rule exists to prevent, and
|
||||
# test_unaudited_host_side_knob_stays_listed_until_it_is_classified (in
|
||||
# tests/test_envs.py) fails on it.
|
||||
#
|
||||
# The fence holds NOTHING BUT the names, one per line. Prose belongs out
|
||||
# here: the gate reads the fence as a set of identifiers, and a knob name
|
||||
# mentioned in passing inside it would otherwise read as a listing.
|
||||
#
|
||||
# The one entry, MERLIN_PIN_BSM, pins Marlin's block_size_m to 64 (Python,
|
||||
# no rebuild). It is an ALTERNATIVE to the split-K toggle above, not a
|
||||
# complement: on its own it fixed only the 74/102-token cells and left
|
||||
# 1050 upward broken, while use_full_k fixed all of them with block_size_m
|
||||
# still varying. The tie-determinism change promotes use_full_k and keeps this one off, so its
|
||||
# read site was never brought across and it stays unclassified.
|
||||
# --8<-- [start:unaudited-host-side-knobs]
|
||||
# MERLIN_PIN_BSM
|
||||
# --8<-- [end:unaudited-host-side-knobs]
|
||||
|
||||
|
||||
def get_default_cache_root():
|
||||
@@ -609,6 +959,47 @@ def _resolve_rust_cli_path() -> str | None:
|
||||
return raw
|
||||
|
||||
|
||||
def _e2_strict_switch(name: str, default: bool = False) -> bool:
|
||||
"""Parse a fork on/off switch that accepts exactly ``"0"`` and ``"1"``.
|
||||
|
||||
Unset means ``default`` (off unless the knob says otherwise). Anything else -- ``"true"``, ``"yes"``, ``""``, ``" 1"``,
|
||||
``"01"`` -- is a boot error: a switch that selects a different transport
|
||||
must not be satisfiable by a spelling nobody meant.
|
||||
"""
|
||||
raw = os.environ.get(name)
|
||||
if raw is None:
|
||||
return default
|
||||
if raw == "0":
|
||||
return False
|
||||
if raw == "1":
|
||||
return True
|
||||
raise ValueError(f'{name} must be exactly "0" or "1" (or unset), got {raw!r}')
|
||||
|
||||
|
||||
def _e2_diagnostic_mode(name: str) -> str:
|
||||
"""Normalize an E2 diagnostic gate to off|warn|raise.
|
||||
|
||||
One normalizer for every knob in the family, so a registry entry and the
|
||||
module that acts on it cannot disagree about what the environment said.
|
||||
``1`` is accepted as ``warn`` for continuity with the earlier boolean
|
||||
gates. A set-but-unrecognised value is a boot error, not a silent ``off``:
|
||||
a broken gate that disarms its own instrument is the failure mode these
|
||||
diagnostics exist to avoid.
|
||||
"""
|
||||
# An EMPTY value means off, not a boot error. Shell scripts and pytest
|
||||
# fixtures both use `VAR=` to mean "unset" (see tests/v1/worker/
|
||||
# test_e2_memstats.py), and rejecting it would turn a routine way of
|
||||
# clearing the variable into a crash.
|
||||
value = os.environ.get(name, "off").strip().lower() or "off"
|
||||
if value == "1":
|
||||
return "warn"
|
||||
if value not in ("off", "warn", "raise"):
|
||||
raise ValueError(
|
||||
f"{name} must be off|warn|raise (or 1 for warn), got {value!r}"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# ================== Installation Time Env Vars ==================
|
||||
# Target device of vLLM, supporting [cuda (by default),
|
||||
@@ -2118,6 +2509,56 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_USE_V2_MODEL_RUNNER": lambda: maybe_convert_bool(
|
||||
os.getenv("VLLM_USE_V2_MODEL_RUNNER", None)
|
||||
),
|
||||
"VLLM_HOST_EMBED_TABLE": lambda: os.getenv("VLLM_HOST_EMBED_TABLE", "") == "1",
|
||||
# Timeout for PLE weight loading and TP worker registration.
|
||||
"VLLM_PLE_OFFLOAD_READY_TIMEOUT": lambda: float(
|
||||
os.getenv("VLLM_PLE_OFFLOAD_READY_TIMEOUT", "600")
|
||||
),
|
||||
# --- Flash-Next fork knobs (v2.2); see the TYPE_CHECKING block for
|
||||
# which of these are compile/capture factors and which are ignored.
|
||||
# Compile/capture factors.
|
||||
"VLLM_PLE_FP8_GLOBAL_SCALE": lambda: (
|
||||
os.getenv("VLLM_PLE_FP8_GLOBAL_SCALE", "") == "1"
|
||||
),
|
||||
"VLLM_QSA_KV_CLIP_COUNT": lambda: os.getenv("VLLM_QSA_KV_CLIP_COUNT", "").strip(),
|
||||
"VLLM_QSA_KV_COLLECT": lambda: os.getenv("VLLM_QSA_KV_COLLECT", "").strip(),
|
||||
# Capture-configuration factor, read in C++; raw passthrough on purpose.
|
||||
"MERLIN_TOPK_EXACT_MEDIUM": lambda: os.getenv("MERLIN_TOPK_EXACT_MEDIUM", None),
|
||||
# PLE pull transport switch; strict "0"/"1" (see the TYPE_CHECKING block).
|
||||
"VLLM_E2_PLE_PULL_TRANSPORT": lambda: _e2_strict_switch(
|
||||
"VLLM_E2_PLE_PULL_TRANSPORT", default=True
|
||||
),
|
||||
# Runtime only; all of the following are in compile_factors.ignored_factors.
|
||||
"VLLM_E2_ALLOW_BREAKABLE": lambda: os.getenv("VLLM_E2_ALLOW_BREAKABLE", "") == "1",
|
||||
"VLLM_E2_EXPECTED_CUDAGRAPH_MODE": lambda: os.getenv(
|
||||
"VLLM_E2_EXPECTED_CUDAGRAPH_MODE", ""
|
||||
).strip(),
|
||||
"VLLM_QSA_KV_SCALES": lambda: os.getenv("VLLM_QSA_KV_SCALES", "").strip(),
|
||||
"VLLM_E2_GUARD_MODE": lambda: (
|
||||
os.getenv("VLLM_E2_GUARD_MODE", "warn").strip().lower()
|
||||
),
|
||||
"VLLM_E2_GUARD_INTERVAL": lambda: os.getenv("VLLM_E2_GUARD_INTERVAL", "0.05"),
|
||||
"VLLM_E2_MEMSTATS": lambda: os.getenv("VLLM_E2_MEMSTATS", "0"),
|
||||
"VLLM_E2_MEMSTATS_DIR": lambda: os.getenv("VLLM_E2_MEMSTATS_DIR", ""),
|
||||
"VLLM_E2_SCHED_LOG": lambda: os.getenv("VLLM_E2_SCHED_LOG", ""),
|
||||
"VLLM_E2_ZERO_OWNERSHIP_CHECK": lambda: (
|
||||
os.getenv("VLLM_E2_ZERO_OWNERSHIP_CHECK", "") == "1"
|
||||
),
|
||||
"VLLM_PLE_OFFLOAD_HOME_DEVICE": lambda: os.getenv(
|
||||
"VLLM_PLE_OFFLOAD_HOME_DEVICE", ""
|
||||
).strip(),
|
||||
"VLLM_PLE_OFFLOAD_WAIT_TIMEOUT": lambda: float(
|
||||
os.getenv("VLLM_PLE_OFFLOAD_WAIT_TIMEOUT", "120")
|
||||
),
|
||||
"VLLM_PLE_PROGRESS_DEADLINE_S": lambda: float(
|
||||
os.getenv("VLLM_PLE_PROGRESS_DEADLINE_S", "300")
|
||||
),
|
||||
"VLLM_PLE_STARTUP_DEADLINE_S": lambda: float(
|
||||
os.getenv("VLLM_PLE_STARTUP_DEADLINE_S", "1800")
|
||||
),
|
||||
"VLLM_PLE_TEST_FAULT": lambda: os.getenv("VLLM_PLE_TEST_FAULT", None),
|
||||
"VLLM_PLE_TEST_FAULT_GATE": lambda: os.getenv("VLLM_PLE_TEST_FAULT_GATE", None),
|
||||
"VLLM_PLE_TEST_FAULT_LOG": lambda: os.getenv("VLLM_PLE_TEST_FAULT_LOG", None),
|
||||
# Log model inspection after loading.
|
||||
# If enabled, logs a transformers-style hierarchical view of the model
|
||||
# with quantization methods and attention backends.
|
||||
@@ -2182,6 +2623,23 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ELASTIC_EP_DRAIN_REQUESTS": lambda: bool(
|
||||
int(os.getenv("VLLM_ELASTIC_EP_DRAIN_REQUESTS", "0"))
|
||||
),
|
||||
# PLE diagnostics. Both are off|warn|raise (1 is accepted as warn) and both
|
||||
# default to off. They are registered here so the registry is the single
|
||||
# source of truth for their normalization -- the modules that use them read
|
||||
# these getters rather than calling os.getenv a second time, so the value
|
||||
# the registry reports cannot drift from the value the check applies.
|
||||
# Both are excluded from compile factors in `compile_factors` below: they
|
||||
# gate HOST-side Python outside any capture and add no node to a compiled
|
||||
# graph. If either ever grows a device-side component, remove it from
|
||||
# `ignored_factors` in the same change.
|
||||
#
|
||||
# Read the pull-transport PLE semaphore back before a real launch, on the
|
||||
# publishing rank only (vllm#53899 one-step-lag class).
|
||||
"VLLM_E2_PLE_FLAG_CHECK": lambda: _e2_diagnostic_mode("VLLM_E2_PLE_FLAG_CHECK"),
|
||||
# Host-side align row-rule and state-block collision check.
|
||||
"VLLM_E2_MAMBA_SLOT_OWNERSHIP": lambda: _e2_diagnostic_mode(
|
||||
"VLLM_E2_MAMBA_SLOT_OWNERSHIP"
|
||||
),
|
||||
# If set to 1, enable CUDA graph memory estimation during memory profiling.
|
||||
# This profiles CUDA graph memory usage to provide more accurate KV cache
|
||||
# memory allocation. Enabled by default as of v0.21.0
|
||||
@@ -2249,6 +2707,60 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_SKIP_VERSION_SUFFIX": lambda: bool(
|
||||
int(os.getenv("VLLM_SKIP_VERSION_SUFFIX", "0"))
|
||||
),
|
||||
# Mamba prefix-checkpoint retention (v2.5; default ON, "0" turns it off).
|
||||
# Strict "0"/"1"; read when each MambaManager is constructed.
|
||||
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT": lambda: _e2_strict_switch(
|
||||
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT", default=True
|
||||
),
|
||||
# Side-cache layout (v2.5; PLE state folded into a GDN group + QSA ring slot
|
||||
# packing), default ON. Strict "0"/"1"; read once per get_kv_cache_configs call.
|
||||
"VLLM_E2_SIDE_CACHE_LAYOUT": lambda: _e2_strict_switch(
|
||||
"VLLM_E2_SIDE_CACHE_LAYOUT", default=True
|
||||
),
|
||||
# Profiling-teardown leak policy: fail the boot unless "0"/"false"/"no".
|
||||
"VLLM_FAIL_ON_PROFILING_KV_LEAK": lambda: os.getenv(
|
||||
"VLLM_FAIL_ON_PROFILING_KV_LEAK", "1"
|
||||
).strip().lower()
|
||||
not in ("0", "false", "no"),
|
||||
# Seconds between `topk-exact-counters` lines from each GPU worker
|
||||
# (0, the default, disables the reader entirely). The line reports the
|
||||
# medium-path exact-restart counters; see
|
||||
# vllm/models/qwen4_exp/nvidia/ops/topk_counters.py.
|
||||
"MERLIN_TOPK_EXACT_COUNTERS_LOG": lambda: int(
|
||||
os.getenv("MERLIN_TOPK_EXACT_COUNTERS_LOG", "0")
|
||||
),
|
||||
# Fallback slot count for the counter reader on a build WITHOUT the
|
||||
# persistent_topk_exact_counters_slots op (pre-tie-determinism). This tree registers
|
||||
# that op, so topk_counters.slot_count() never consults the variable here.
|
||||
# Raw passthrough: the reader's int(os.getenv(..., "5")) is the parser.
|
||||
"MERLIN_TOPK_COUNTER_SLOTS": lambda: os.getenv("MERLIN_TOPK_COUNTER_SLOTS", None),
|
||||
# Determinism toggles. See the declaration block above for the AOT /
|
||||
# capture classification and for why three of these are raw passthroughs.
|
||||
# All four are absent from ignored_factors on purpose, so all four are
|
||||
# hashed into the compile cache key.
|
||||
"MERLIN_FULL_K": lambda: os.getenv("MERLIN_FULL_K", None),
|
||||
"MERLIN_TIE_DET": lambda: os.getenv("MERLIN_TIE_DET", None),
|
||||
"MERLIN_TIE_RECENT": lambda: os.getenv("MERLIN_TIE_RECENT", None),
|
||||
"MERLIN_QSA_SORT": lambda: bool(int(os.getenv("MERLIN_QSA_SORT", "0"))),
|
||||
# persistent_topk kernel switches. Raw passthroughs
|
||||
# of values C++ parses; see the declaration block above.
|
||||
# MERLIN_TOPK_EXACT_MEDIUM is registered once, earlier in this dict.
|
||||
"MERLIN_TOPK_EXACT_DECODE": lambda: os.getenv("MERLIN_TOPK_EXACT_DECODE", None),
|
||||
"MERLIN_TOPK_DET_LARGE": lambda: os.getenv("MERLIN_TOPK_DET_LARGE", None),
|
||||
# Large-path fault injectors. TEST ONLY, hashed for
|
||||
# the reason in the declaration block: both are capture-baked.
|
||||
"MERLIN_TOPK_LARGE_FAULT_INJECT": lambda: os.getenv(
|
||||
"MERLIN_TOPK_LARGE_FAULT_INJECT", None
|
||||
),
|
||||
"MERLIN_TOPK_LARGE_FAULT_KIND": lambda: os.getenv(
|
||||
"MERLIN_TOPK_LARGE_FAULT_KIND", None
|
||||
),
|
||||
# In-kernel canonical emission and its debug check; see the
|
||||
# declaration block. Both hashed: both change the captured sequence.
|
||||
"MERLIN_TOPK_SORTED_EMIT": lambda: os.getenv("MERLIN_TOPK_SORTED_EMIT", None),
|
||||
"MERLIN_TOPK_SORTED_EMIT_CHECK": lambda: bool(
|
||||
int(os.getenv("MERLIN_TOPK_SORTED_EMIT_CHECK", "0"))
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -2405,6 +2917,77 @@ def compile_factors() -> dict[str, object]:
|
||||
"CUDA_VISIBLE_DEVICES",
|
||||
"NO_COLOR",
|
||||
"VLLM_SKIP_VERSION_SUFFIX",
|
||||
# --- Flash-Next fork knobs that are runtime-only (v2.2) ------
|
||||
# Registering a name in environment_variables makes it a compile
|
||||
# factor by default (this function is a deny-list), so every fork knob
|
||||
# that cannot change a compiled or captured artifact is listed here
|
||||
# with its reason. Hashing them would invalidate the whole AOT cache on
|
||||
# a log path, a timeout or a guard-mode flip.
|
||||
#
|
||||
# Static QSA K/V scales: fill_()ed into existing fp32 device buffers
|
||||
# after load and before capture; the kernel reads them with
|
||||
# tl.load(k_scale_ptr), never as a constexpr.
|
||||
"VLLM_QSA_KV_SCALES",
|
||||
# Guard telemetry: counter() returns a real [3] int64 view in every
|
||||
# mode, so the Triton arguments and the captured graph are identical;
|
||||
# the mode only gates the side reader thread and the check_pending
|
||||
# raise, and the interval is that thread's poll cadence.
|
||||
"VLLM_E2_GUARD_MODE",
|
||||
"VLLM_E2_GUARD_INTERVAL",
|
||||
# Debug telemetry cadence and output locations.
|
||||
"VLLM_E2_MEMSTATS",
|
||||
"VLLM_E2_MEMSTATS_DIR",
|
||||
"VLLM_E2_SCHED_LOG",
|
||||
# Host-side scheduler assertion; the scheduler is never compiled.
|
||||
"VLLM_E2_ZERO_OWNERSHIP_CHECK",
|
||||
# Placement only: rewrites CUDA_VISIBLE_DEVICES for the PLE offload
|
||||
# child, and CUDA_VISIBLE_DEVICES is already ignored above.
|
||||
"VLLM_PLE_OFFLOAD_HOME_DEVICE",
|
||||
# Timeouts and deadlines, in seconds. VLLM_PLE_OFFLOAD_READY_TIMEOUT
|
||||
# was registered in the v2 host-embed commit and left hashed by
|
||||
# oversight: serve-v2.sh sets it to 1800, so every boot that changed it
|
||||
# missed the AOT cache for no reason.
|
||||
"VLLM_PLE_OFFLOAD_READY_TIMEOUT",
|
||||
"VLLM_PLE_OFFLOAD_WAIT_TIMEOUT",
|
||||
"VLLM_PLE_PROGRESS_DEADLINE_S",
|
||||
"VLLM_PLE_STARTUP_DEADLINE_S",
|
||||
# Test-only fault injection: a cut spec, a marker log path and a
|
||||
# bootstrap gate file. None of them reaches a kernel or a graph.
|
||||
"VLLM_PLE_TEST_FAULT",
|
||||
"VLLM_PLE_TEST_FAULT_GATE",
|
||||
"VLLM_PLE_TEST_FAULT_LOG",
|
||||
# Fork startup guard: the opt-out permits a different identity and the
|
||||
# expected-mode pin asserts one; neither IS one.
|
||||
"VLLM_E2_ALLOW_BREAKABLE",
|
||||
"VLLM_E2_EXPECTED_CUDAGRAPH_MODE",
|
||||
# INERT on the 0.30 base: no read site, so it cannot change any artifact.
|
||||
# Hashing it would invalidate caches for no behavioural change. Restore
|
||||
# it to the compile factors in the same commit that relands its reader.
|
||||
"VLLM_PLE_FP8_GLOBAL_SCALE",
|
||||
# Diagnostic knobs, both strictly host-side and outside capture, so
|
||||
# neither AOT nor capture identity depends on them. Hashing
|
||||
# them would invalidate every compile cache the moment a diagnostic
|
||||
# boot is taken -- exactly when the cache is wanted. Remove from this
|
||||
# set if either ever grows a device-side component.
|
||||
# MERLIN_TOPK_EXACT_MEDIUM is deliberately NOT here: it is a capture
|
||||
# configuration factor and is hashed conservatively.
|
||||
"VLLM_E2_PLE_FLAG_CHECK",
|
||||
"VLLM_E2_MAMBA_SLOT_OWNERSHIP",
|
||||
# Scheduler-side block retention (v2.5). The scheduler is never
|
||||
# compiled and the worker's tables are identical in both worlds, so an
|
||||
# A/B flip must reuse the same AOT cache.
|
||||
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT",
|
||||
# Boot-failure policy of a host-side profiling-teardown check; read by no
|
||||
# compiled code.
|
||||
"VLLM_FAIL_ON_PROFILING_KV_LEAK",
|
||||
# Logging cadence for an already-compiled kernel's counters: it gates a
|
||||
# host-side timer and a snapshot op, and is read by no compiled code, so
|
||||
# hashing it would invalidate every compile cache for a log interval.
|
||||
"MERLIN_TOPK_EXACT_COUNTERS_LOG",
|
||||
# Host-side reader fallback (buffer length for a snapshot op outside
|
||||
# capture); dead on a build that answers the _slots query.
|
||||
"MERLIN_TOPK_COUNTER_SLOTS",
|
||||
# ------------------------------------------------------------------
|
||||
"FORCE_COLOR",
|
||||
}
|
||||
|
||||
|
||||
@@ -562,16 +562,18 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
elif current_platform.is_xpu():
|
||||
self.gdn_decode_kernel = "XPU"
|
||||
|
||||
self.enable_fused_gdn_spec_decode = (
|
||||
self.gdn_decode_kernel == "cuda" and self.speculative_config is not None
|
||||
)
|
||||
# RecoverSSM verify replaces the per-draft snapshot path (Triton decode
|
||||
# only)
|
||||
self.use_gdn_recoverssm = self._uses_gdn_recoverssm()
|
||||
(
|
||||
self.gdn_decode_kernel,
|
||||
self.enable_fused_gdn_spec_decode,
|
||||
self.enable_packed_recurrent_decode,
|
||||
) = self._decode_paths(
|
||||
self.gdn_decode_kernel,
|
||||
self.use_gdn_recoverssm,
|
||||
self.enable_packed_recurrent_decode,
|
||||
self.speculative_config is not None,
|
||||
)
|
||||
if self.use_gdn_recoverssm:
|
||||
self.gdn_decode_kernel = "triton"
|
||||
self.enable_fused_gdn_spec_decode = False
|
||||
self.enable_packed_recurrent_decode = False
|
||||
logger.info_once(
|
||||
"GDN RecoverSSM speculative verify active (spec_query_len %d)",
|
||||
self.num_spec + 1,
|
||||
@@ -584,6 +586,29 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
@staticmethod
|
||||
def _decode_paths(
|
||||
gdn_decode_kernel: str,
|
||||
use_recoverssm: bool,
|
||||
packed_recurrent_decode: bool,
|
||||
has_speculative_config: bool,
|
||||
) -> tuple[str, bool, bool]:
|
||||
"""(decode kernel, fused spec-decode forward, packed recurrent decode).
|
||||
|
||||
The fused forward is on whenever the CUDA decode kernel is eligible and
|
||||
the engine runs speculation (#59735 keeps non-speculative decode on the
|
||||
standard path). RecoverSSM's verify replaces the per-draft snapshot path
|
||||
and needs the Triton decode kernel for its spec rows, so with RecoverSSM
|
||||
the decode kernel is Triton and the packed recurrent decode is off. The
|
||||
fused forward stays on: it calls _forward_core for prefill and verify
|
||||
rows alike and only fuses the input split and the gated norm. Turning it
|
||||
off sends every prefill chunk down the unfused path.
|
||||
"""
|
||||
enable_fused_forward = gdn_decode_kernel == "cuda" and has_speculative_config
|
||||
if use_recoverssm:
|
||||
return "triton", enable_fused_forward, False
|
||||
return gdn_decode_kernel, enable_fused_forward, packed_recurrent_decode
|
||||
|
||||
def _fused_gdn_decode_unsupported_reason(
|
||||
self, vllm_config: VllmConfig
|
||||
) -> str | None:
|
||||
@@ -1453,7 +1478,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
if (
|
||||
spec_sequence_masks is not None
|
||||
and mixed_qkv_spec is not None
|
||||
and self.use_gdn_recoverssm
|
||||
and getattr(self, "use_gdn_recoverssm", False)
|
||||
):
|
||||
query_spec, key_spec, value_spec, verify_views = (
|
||||
self._recoverssm_verify_qkv(mixed_qkv_spec)
|
||||
|
||||
@@ -9,7 +9,7 @@ GDN port of the Kimi-K3 KDA RecoverSSM path
|
||||
scalar per value head (g = -exp(A_log) * softplus(a + dt_bias)), q/k heads are grouped
|
||||
over value heads (HV // H), and q/k are L2-normalised inside the kernel when requested.
|
||||
The verify step caches, per token, the delta-rule correction c_t = beta_t * (v_t - S'_t
|
||||
k_t) (V floats), the normalised key (K floats) and the log-decay g_t (1 float) in one
|
||||
k_t) (V floats), the normalised key (K floats) and the decay exp(g_t) (1 float) in one
|
||||
fp32 "replay" record. Recovery folds the accepted prefix: S_n = exp(sum_t g_t) S_0 +
|
||||
sum_t exp(sum_{s>t} g_s) c_t k_t^T and writes the checkpoint once (plus the
|
||||
block-boundary state in align mode)."""
|
||||
@@ -150,7 +150,8 @@ def _gdn_recoverssm_verify_kernel(
|
||||
q = q * tl.rsqrt(tl.sum(q * q) + 1e-6)
|
||||
k = k * tl.rsqrt(tl.sum(k * k) + 1e-6)
|
||||
q = q * scale
|
||||
new_state = state * tl.exp(g)
|
||||
decay = tl.exp(g)
|
||||
new_state = state * decay
|
||||
corr = (v - tl.sum(new_state * k[None, :], 1)) * beta
|
||||
new_state = new_state + corr[:, None] * k[None, :]
|
||||
state = tl.where(valid, new_state, state)
|
||||
@@ -164,7 +165,8 @@ def _gdn_recoverssm_verify_kernel(
|
||||
tl.store(rec + offs_v * stride_replay_dim, corr, mask=valid & mask_v)
|
||||
if pid_v == 0:
|
||||
tl.store(rec + (V + offs_k) * stride_replay_dim, k, mask=valid & mask_k)
|
||||
tl.store(rec + (V + K) * stride_replay_dim, g, mask=valid)
|
||||
# the decay as multiplied here, so the commit replays this exact float
|
||||
tl.store(rec + (V + K) * stride_replay_dim, decay, mask=valid)
|
||||
|
||||
|
||||
def gdn_recoverssm_verify(
|
||||
@@ -351,8 +353,12 @@ def _commit_gdn_state_kernel(
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
NUM_HEADS: tl.constexpr,
|
||||
ALIGN_MODE: tl.constexpr,
|
||||
BOUNDARY: tl.constexpr,
|
||||
):
|
||||
# Every program runs one fold and keeps one tile live. In align mode the
|
||||
# block-boundary state (the window's first boundary_len tokens) is folded by a
|
||||
# second launch (BOUNDARY): a crossing window's boundary state goes into its
|
||||
# source block, which the first launch's final-state fold must still read.
|
||||
pid_v = tl.program_id(0)
|
||||
pid_b = tl.program_id(1)
|
||||
pid_lh = tl.program_id(2)
|
||||
@@ -365,10 +371,16 @@ def _commit_gdn_state_kernel(
|
||||
if n == 0:
|
||||
return
|
||||
final_idx = tl.load(final_state_indices_ptr + pid_b).to(tl.int64)
|
||||
boundary_idx = tl.load(boundary_state_indices_ptr + pid_b).to(tl.int64)
|
||||
boundary_len = tl.load(boundary_recovery_lens_ptr + pid_b)
|
||||
if final_idx <= null_block_id:
|
||||
return
|
||||
dst_idx = final_idx
|
||||
if BOUNDARY:
|
||||
dst_idx = tl.load(boundary_state_indices_ptr + pid_b).to(tl.int64)
|
||||
# A window ending exactly on the boundary keeps its final state in that
|
||||
# block: the first launch already wrote it (and may have overwritten src).
|
||||
if dst_idx <= null_block_id or dst_idx == final_idx:
|
||||
return
|
||||
n = tl.load(boundary_recovery_lens_ptr + pid_b)
|
||||
state_ptr = tl.load(state_base_addrs_ptr + pid_l).to(
|
||||
tl.pointer_type(state_ref_ptr.dtype.element_ty)
|
||||
)
|
||||
@@ -389,35 +401,19 @@ def _commit_gdn_state_kernel(
|
||||
mask=mask_state,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
fin_decay = 1.0
|
||||
fin_corr = tl.zeros([BV, BK], tl.float32)
|
||||
bnd_decay = 1.0
|
||||
bnd_corr = tl.zeros([BV, BK], tl.float32)
|
||||
for r in range(n):
|
||||
t = n - r - 1
|
||||
# Forward over the accepted tokens with the verify step's own update, so the committed
|
||||
# state is the per-token state bit for bit (the decay and corr stored are the floats
|
||||
# the verify kernel used).
|
||||
h = s0
|
||||
for t in range(n):
|
||||
rec = rec_base + t * stride_replay_pos
|
||||
c = tl.load(rec + offs_v * stride_replay_dim, mask=mask_v, other=0.0)
|
||||
kk = tl.load(rec + (V + offs_k) * stride_replay_dim, mask=mask_k, other=0.0)
|
||||
g = tl.load(rec + (V + K) * stride_replay_dim)
|
||||
upd = c[:, None] * kk[None, :]
|
||||
dec = tl.exp(g)
|
||||
fin_corr += upd * fin_decay
|
||||
fin_decay *= dec
|
||||
if ALIGN_MODE:
|
||||
before = t < boundary_len
|
||||
bnd_corr += tl.where(before, upd * bnd_decay, 0.0)
|
||||
bnd_decay *= tl.where(before, dec, 1.0)
|
||||
if ALIGN_MODE:
|
||||
bs = s0 * bnd_decay + bnd_corr
|
||||
tl.store(
|
||||
state_ptr + boundary_idx * sbs + pid_h * stride_state_head + sp,
|
||||
bs.to(state_ref_ptr.dtype.element_ty),
|
||||
mask=mask_state & (boundary_idx > null_block_id),
|
||||
)
|
||||
fs = s0 * fin_decay + fin_corr
|
||||
decay = tl.load(rec + (V + K) * stride_replay_dim)
|
||||
h = tl.fma(c[:, None], kk[None, :], h * decay)
|
||||
tl.store(
|
||||
state_ptr + final_idx * sbs + pid_h * stride_state_head + sp,
|
||||
fs.to(state_ref_ptr.dtype.element_ty),
|
||||
state_ptr + dst_idx * sbs + pid_h * stride_state_head + sp,
|
||||
h.to(state_ref_ptr.dtype.element_ty),
|
||||
mask=mask_state,
|
||||
)
|
||||
|
||||
@@ -659,9 +655,12 @@ class GDNRecoverSSMCommitContext:
|
||||
ref = self.checkpoints[0]
|
||||
_, HV, V, K = ref.shape
|
||||
BK = triton.next_power_of_2(K)
|
||||
BV = min(triton.next_power_of_2(V), 32)
|
||||
# 16-row tiles: the commit streams each state once, and smaller tiles
|
||||
# keep more of them in flight (about 13% more bandwidth than 32 on H200).
|
||||
BV = min(triton.next_power_of_2(V), 16)
|
||||
rr = self.replays[0]
|
||||
_commit_gdn_state_kernel[(triton.cdiv(V, BV), batch, num_layers * HV)](
|
||||
align = block_table is not None
|
||||
args = (
|
||||
ref,
|
||||
self.state_base_addrs,
|
||||
self.state_block_strides,
|
||||
@@ -681,15 +680,22 @@ class GDNRecoverSSMCommitContext:
|
||||
rr.stride(2),
|
||||
rr.stride(3),
|
||||
state_indices.stride(0),
|
||||
K=K,
|
||||
V=V,
|
||||
BK=BK,
|
||||
BV=BV,
|
||||
NUM_HEADS=HV,
|
||||
ALIGN_MODE=block_table is not None,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
grid = (triton.cdiv(V, BV), batch, num_layers * HV)
|
||||
# Final states first: a boundary state can land in the window's source
|
||||
# slot, which the final-state fold reads (stream order separates them).
|
||||
for boundary in (False, True) if align else (False,):
|
||||
_commit_gdn_state_kernel[grid](
|
||||
*args,
|
||||
K=K,
|
||||
V=V,
|
||||
BK=BK,
|
||||
BV=BV,
|
||||
NUM_HEADS=HV,
|
||||
BOUNDARY=boundary,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["GDNRecoverSSMCommitContext", "gdn_recoverssm_verify"]
|
||||
|
||||
@@ -0,0 +1,563 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Base class for PLE layers that support cross-process CPU offload.
|
||||
|
||||
Synchronization between the offload process and the GPU main stream is
|
||||
handled by :class:`CpuGpuSemaphore`, which uses CUDA's
|
||||
``cuStreamWaitValue32`` / ``cuStreamWriteValue32`` stream memory operations.
|
||||
|
||||
Decode-step timeline, host-mapped PULL design
|
||||
--------------------------------------------------------
|
||||
Offload process (CPU, no CUDA) GPU main stream (forward)
|
||||
------------------------------ ------------------------------
|
||||
forward_impl() on CPU ... other GPU ops ...
|
||||
wait host flag == 0 WaitValue32(host flag == 1) <- in graph
|
||||
rows -> shared pinned host slot cuMemcpyHtoDAsync(slot -> GPU) <- in graph
|
||||
publish fence; host flag = 1 return _gpu_output_buffer[:n]
|
||||
... model consumes output ...
|
||||
WriteValue32(host flag = 0) <- release_outputs
|
||||
|
||||
``release_outputs`` performs the sole reset. Safety requires serialized local
|
||||
forwards on one ordered model stream: a new transaction cannot begin until the
|
||||
prior release has been enqueued. If any prepared forward is abandoned or its
|
||||
correspondence becomes uncertain, the worker must terminate; resetting,
|
||||
retrying, or continuing could reuse stale rows or overwrite live input.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import functools
|
||||
import threading
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterator
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
from cuda.bindings import driver as cuda_driver
|
||||
from cuda.bindings.driver import CUstreamWaitValue_flags
|
||||
from torch import nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
# Module-level flag set to True inside the offload subprocess.
|
||||
# Because the offload process and GPU worker processes are separate OS
|
||||
# processes (spawned via multiprocessing), each has its own memory space.
|
||||
# A plain module-level bool is sufficient -- no thread-local storage needed.
|
||||
_offload_worker_flag = False
|
||||
|
||||
# Native FULL capture runs on one Python thread. The custom-op implementation
|
||||
# appends the exact arguments from every wait-copy invocation while that thread
|
||||
# is inside the graph manager's capture scope.
|
||||
_ple_copy_capture = threading.local()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def observe_ple_copy_geometry() -> Iterator[list[tuple[str, int, int]]]:
|
||||
"""Collect ``(layer, rows, bytes)`` from actual wait-copy executions."""
|
||||
if getattr(_ple_copy_capture, "records", None) is not None:
|
||||
raise RuntimeError("nested PLE copy-geometry observation is unsupported")
|
||||
records: list[tuple[str, int, int]] = []
|
||||
_ple_copy_capture.records = records
|
||||
try:
|
||||
yield records
|
||||
finally:
|
||||
del _ple_copy_capture.records
|
||||
|
||||
|
||||
def _record_ple_copy_geometry(
|
||||
layer_name: str,
|
||||
num_tokens: int,
|
||||
row_bytes: int,
|
||||
) -> None:
|
||||
records = getattr(_ple_copy_capture, "records", None)
|
||||
if records is not None:
|
||||
rows = int(num_tokens)
|
||||
records.append((str(layer_name), rows, rows * int(row_bytes)))
|
||||
|
||||
|
||||
def is_offload_process() -> bool:
|
||||
"""Return True inside the dedicated PLE CPU-offload subprocess."""
|
||||
return _offload_worker_flag
|
||||
|
||||
|
||||
def mark_as_offload_worker() -> None:
|
||||
"""Mark the current process as the dedicated PLE offload worker."""
|
||||
global _offload_worker_flag
|
||||
_offload_worker_flag = True
|
||||
|
||||
|
||||
def _cuda_check(result: Any, operation: str) -> Any:
|
||||
"""Check the ``(CUresult, ...)`` tuple returned by cuda-python calls."""
|
||||
error = result[0] if isinstance(result, tuple) else result
|
||||
if error.value != 0:
|
||||
raise RuntimeError(f"{operation} failed: {error}")
|
||||
return result
|
||||
|
||||
|
||||
class CpuGpuSemaphore:
|
||||
"""Cross-process semaphore backed by a one-element int32 CUDA tensor.
|
||||
|
||||
The flag is stored in a regular GPU tensor that is shared through
|
||||
PyTorch's CUDA IPC mechanism. Both processes obtain device pointers that
|
||||
map to the same physical GPU memory, so a stream-memory wait in the GPU
|
||||
worker observes the write issued by the offload process.
|
||||
"""
|
||||
|
||||
RESET_VALUE = 0
|
||||
DONE_VALUE = 1
|
||||
|
||||
def __init__(self, device: torch.device) -> None:
|
||||
self._flag_tensor = torch.zeros(1, dtype=torch.int32, device=device)
|
||||
self._flag_ptr: int | None = None
|
||||
|
||||
@classmethod
|
||||
def from_ipc_tensor(cls, flag_tensor: torch.Tensor) -> "CpuGpuSemaphore":
|
||||
"""Construct a semaphore from a CUDA tensor received through IPC."""
|
||||
semaphore = cls.__new__(cls)
|
||||
semaphore._flag_tensor = flag_tensor
|
||||
semaphore._flag_ptr = None
|
||||
return semaphore
|
||||
|
||||
@classmethod
|
||||
def from_host_mapped(
|
||||
cls, host_flag_tensor: torch.Tensor, device_ptr: int
|
||||
) -> "CpuGpuSemaphore":
|
||||
"""Semaphore on a page-locked, device-mapped shared host int32.
|
||||
|
||||
``host_flag_tensor`` keeps the shared storage alive; ``device_ptr`` is
|
||||
its device address (``cuMemHostGetDevicePointer``). Stream memory ops
|
||||
on it are executed by the GPU against host memory, so a CPU process
|
||||
with no CUDA context can signal it with a plain store.
|
||||
"""
|
||||
semaphore = cls.__new__(cls)
|
||||
semaphore._flag_tensor = host_flag_tensor
|
||||
semaphore._flag_ptr = int(device_ptr)
|
||||
return semaphore
|
||||
|
||||
@property
|
||||
def flag_tensor(self) -> torch.Tensor:
|
||||
"""Return the tensor that backs the semaphore flag."""
|
||||
return self._flag_tensor
|
||||
|
||||
@property
|
||||
def flag_ptr(self) -> int:
|
||||
"""Device address used by the stream memory operations."""
|
||||
if self._flag_ptr is not None:
|
||||
return self._flag_ptr
|
||||
return self._flag_tensor.data_ptr()
|
||||
|
||||
def reset(self, stream: torch.cuda.Stream | None = None) -> None:
|
||||
"""Enqueue ``WriteValue32(flag=0)`` on ``stream``."""
|
||||
if stream is None:
|
||||
stream = torch.cuda.current_stream()
|
||||
_cuda_check(
|
||||
cuda_driver.cuStreamWriteValue32(
|
||||
cuda_driver.CUstream(stream.cuda_stream),
|
||||
cuda_driver.CUdeviceptr(self.flag_ptr),
|
||||
self.RESET_VALUE,
|
||||
0,
|
||||
),
|
||||
"CpuGpuSemaphore.reset",
|
||||
)
|
||||
|
||||
def signal(self, stream: torch.cuda.Stream | None = None) -> None:
|
||||
"""Enqueue ``WriteValue32(flag=1)`` on ``stream``."""
|
||||
if stream is None:
|
||||
stream = torch.cuda.current_stream()
|
||||
_cuda_check(
|
||||
cuda_driver.cuStreamWriteValue32(
|
||||
cuda_driver.CUstream(stream.cuda_stream),
|
||||
cuda_driver.CUdeviceptr(self.flag_ptr),
|
||||
self.DONE_VALUE,
|
||||
0,
|
||||
),
|
||||
"CpuGpuSemaphore.signal",
|
||||
)
|
||||
|
||||
def wait_reset(self, stream: torch.cuda.Stream | None = None) -> None:
|
||||
"""Enqueue ``WaitValue32(flag==0)`` on ``stream``."""
|
||||
if stream is None:
|
||||
stream = torch.cuda.current_stream()
|
||||
_cuda_check(
|
||||
cuda_driver.cuStreamWaitValue32(
|
||||
cuda_driver.CUstream(stream.cuda_stream),
|
||||
cuda_driver.CUdeviceptr(self.flag_ptr),
|
||||
self.RESET_VALUE,
|
||||
CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ.value,
|
||||
),
|
||||
"CpuGpuSemaphore.wait_reset",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Custom op: vllm::ple_offload_wait
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wraps cuStreamWaitValue32 as a torch custom op so that
|
||||
# torch.compile(fullgraph=True) treats the CUDA Driver call as an opaque node.
|
||||
# The hidden_states argument creates a dynamo data-dependency edge that prevents
|
||||
# the wait from being reordered before preceding GPU work.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _ple_offload_wait_impl(
|
||||
sem_flag_tensor: torch.Tensor,
|
||||
gpu_output_buffer: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> None:
|
||||
"""Wait for the CPU result without releasing its output buffer."""
|
||||
stream = torch.cuda.current_stream()
|
||||
cuda_stream = cuda_driver.CUstream(stream.cuda_stream)
|
||||
dev_ptr = cuda_driver.CUdeviceptr(sem_flag_tensor.data_ptr())
|
||||
_cuda_check(
|
||||
cuda_driver.cuStreamWaitValue32(
|
||||
cuda_stream,
|
||||
dev_ptr,
|
||||
CpuGpuSemaphore.DONE_VALUE,
|
||||
CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ.value,
|
||||
),
|
||||
"cuStreamWaitValue32(done)",
|
||||
)
|
||||
|
||||
|
||||
def _ple_offload_wait_fake(
|
||||
sem_flag_tensor: torch.Tensor,
|
||||
gpu_output_buffer: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> None:
|
||||
"""Represent the side-effect-only wait during dynamo tracing."""
|
||||
pass
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="ple_offload_wait",
|
||||
op_func=_ple_offload_wait_impl,
|
||||
mutates_args=["gpu_output_buffer"],
|
||||
fake_impl=_ple_offload_wait_fake,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Custom op: vllm::ple_offload_wait_copy (host-mapped pull design)
|
||||
# ---------------------------------------------------------------------------
|
||||
# The offload process writes the result into a page-locked shared host buffer
|
||||
# and stores flag=1 in a page-locked shared host int32; it holds no CUDA
|
||||
# context. On the CURRENT stream (captured into the graph like the wait above):
|
||||
# cuStreamWaitValue32(host flag == 1) -> GPU polls host memory
|
||||
# cuMemcpyAsync(gpu buffer <- host buffer, num_tokens * row_bytes)
|
||||
# The copy engine reads host memory coherently after the flag was observed,
|
||||
# and the consumers read device memory as before. The flag is NOT reset here:
|
||||
# ``release_outputs`` resets it after the forward (unchanged semantics), so
|
||||
# capture warm-ups and dummy runs behave exactly as with ``ple_offload_wait``.
|
||||
# No CPU tensor and no raw address enters the compiled graph: the op resolves
|
||||
# the live pointers by layer name (see _PULL_LAYERS).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Fork (found on the reference host): the host flag / host output pointers and
|
||||
# the row size are NOT op arguments any more. As Python ints they were
|
||||
# specialised into the compiled graph as constants, so a boot that LOADED the
|
||||
# AOT compile cache (no Dynamo pass) waited on the previous process's addresses:
|
||||
# cuStreamWaitValue32 -> CUDA_ERROR_INVALID_VALUE on the first forward. The op
|
||||
# now carries only process-stable values (buffer, token count, layer name) and
|
||||
# resolves the live pointers from this registry on every call.
|
||||
_PULL_LAYERS: dict[str, "PleOffloadLayer"] = {}
|
||||
|
||||
|
||||
def _ple_offload_wait_copy_impl(
|
||||
gpu_output_buffer: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
num_tokens: int,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""Resolve this process's live pointers for ``layer_name``, then wait+DMA."""
|
||||
layer = _PULL_LAYERS.get(layer_name)
|
||||
if layer is None or layer._host_output_ptr is None:
|
||||
raise RuntimeError(
|
||||
f"PLE wait_copy: no armed pull layer named {layer_name!r} in this "
|
||||
f"process (armed: {sorted(_PULL_LAYERS)})"
|
||||
)
|
||||
_ple_offload_wait_copy_raw(
|
||||
gpu_output_buffer,
|
||||
hidden_states,
|
||||
layer._sem.flag_ptr,
|
||||
layer._host_output_ptr,
|
||||
layer._host_row_bytes,
|
||||
num_tokens,
|
||||
layer_name,
|
||||
)
|
||||
|
||||
|
||||
def _ple_offload_wait_copy_raw(
|
||||
gpu_output_buffer: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
flag_ptr: int,
|
||||
host_ptr: int,
|
||||
row_bytes: int,
|
||||
num_tokens: int,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""Wait for the host-side result, then DMA it into the GPU buffer."""
|
||||
if not 0 <= int(num_tokens) <= gpu_output_buffer.shape[0] or int(row_bytes) != (
|
||||
gpu_output_buffer.shape[1] * gpu_output_buffer.element_size()
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"PLE wait_copy out of bounds: {num_tokens} rows x {row_bytes} B vs "
|
||||
f"GPU buffer {tuple(gpu_output_buffer.shape)} {gpu_output_buffer.dtype}"
|
||||
)
|
||||
nbytes = int(num_tokens) * int(row_bytes)
|
||||
_record_ple_copy_geometry(layer_name, num_tokens, row_bytes)
|
||||
stream = cuda_driver.CUstream(torch.cuda.current_stream().cuda_stream)
|
||||
_cuda_check(
|
||||
cuda_driver.cuStreamWaitValue32(
|
||||
stream,
|
||||
cuda_driver.CUdeviceptr(flag_ptr),
|
||||
CpuGpuSemaphore.DONE_VALUE,
|
||||
CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ.value,
|
||||
),
|
||||
"cuStreamWaitValue32(host flag done)",
|
||||
)
|
||||
if nbytes > 0:
|
||||
# host_ptr is this worker's host address of page-locked (registered)
|
||||
# memory, so the copy is a true async DMA and is capturable.
|
||||
_cuda_check(
|
||||
cuda_driver.cuMemcpyHtoDAsync(
|
||||
cuda_driver.CUdeviceptr(gpu_output_buffer.data_ptr()),
|
||||
host_ptr,
|
||||
nbytes,
|
||||
stream,
|
||||
),
|
||||
"cuMemcpyHtoDAsync(PLE host -> GPU)",
|
||||
)
|
||||
|
||||
|
||||
def _ple_offload_wait_copy_fake(
|
||||
gpu_output_buffer: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
num_tokens: int,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""Represent the side-effect-only wait + copy during dynamo tracing."""
|
||||
pass
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="ple_offload_wait_copy",
|
||||
op_func=_ple_offload_wait_copy_impl,
|
||||
mutates_args=["gpu_output_buffer"],
|
||||
fake_impl=_ple_offload_wait_copy_fake,
|
||||
)
|
||||
|
||||
|
||||
class PleOffloadLayer(nn.Module, ABC):
|
||||
"""Base class for embedding-like PLE layers that can run on CPU.
|
||||
|
||||
Subclasses implement :meth:`forward_impl`, which contains the regular
|
||||
on-device computation. In a GPU worker with PLE offload enabled, subclass
|
||||
constructors are skipped so their large weights are never allocated. The
|
||||
CPU process constructs a complete copy and owns those weights instead.
|
||||
"""
|
||||
|
||||
_is_cpu_offloaded = False
|
||||
_gpu_output_buffer: torch.Tensor
|
||||
_sem: CpuGpuSemaphore
|
||||
# Host-mapped pull design: host address of the shared output buffer and
|
||||
# the bytes per output row. ``None`` keeps the IPC push behaviour.
|
||||
_host_output_ptr: int | None = None
|
||||
_host_row_bytes: int = 0
|
||||
_ple_offload_name: str = "<unbound>"
|
||||
|
||||
def __init_subclass__(cls, **kwargs: object) -> None:
|
||||
"""Skip subclass initialization in cross-process GPU-worker mode."""
|
||||
super().__init_subclass__(**kwargs)
|
||||
original_init_obj = cls.__dict__.get("__init__")
|
||||
if original_init_obj is None or not callable(original_init_obj):
|
||||
return
|
||||
original_init = cast(Callable[..., None], original_init_obj)
|
||||
|
||||
@functools.wraps(original_init)
|
||||
def guarded_init(
|
||||
self: "PleOffloadLayer", *args: object, **kwargs: object
|
||||
) -> None:
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT and not is_offload_process():
|
||||
nn.Module.__init__(self)
|
||||
return
|
||||
original_init(self, *args, **kwargs)
|
||||
|
||||
cls.__init__ = guarded_init # type: ignore[method-assign, assignment]
|
||||
|
||||
@classmethod
|
||||
def get_target_device(cls) -> torch.device:
|
||||
"""Return CPU for the offload process and the active GPU otherwise."""
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
return torch.device("cpu")
|
||||
return torch.device("cuda", torch.accelerator.current_device_index())
|
||||
|
||||
@abstractmethod
|
||||
def forward_impl(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
*args: object,
|
||||
**kwargs: object,
|
||||
) -> torch.Tensor:
|
||||
"""Execute the actual embedding computation on the owning device."""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_offload_output_dtype(self, default_dtype: torch.dtype) -> torch.dtype:
|
||||
"""Return the dtype used by cross-process output buffers."""
|
||||
return default_dtype
|
||||
|
||||
def get_offload_output_dim(self, default_dim: int) -> int:
|
||||
"""Return the last dimension used by cross-process output buffers."""
|
||||
return default_dim
|
||||
|
||||
def initialize_dummy_offload_metadata(self, device: torch.device) -> None:
|
||||
"""Initialize metadata needed by dummy offload execution."""
|
||||
|
||||
def setup_cross_process_offload(
|
||||
self,
|
||||
gpu_output_buffer: torch.Tensor,
|
||||
semaphore: CpuGpuSemaphore,
|
||||
host_output_ptr: int | None = None,
|
||||
host_row_bytes: int = 0,
|
||||
) -> None:
|
||||
"""Configure the GPU-worker placeholder with its offload resources."""
|
||||
self._is_cpu_offloaded = True
|
||||
self._gpu_output_buffer = gpu_output_buffer
|
||||
self._sem = semaphore
|
||||
self._host_output_ptr = host_output_ptr
|
||||
self._host_row_bytes = int(host_row_bytes)
|
||||
if host_output_ptr is not None:
|
||||
# Hardening: the registry key is the name the
|
||||
# traced graph carries, so it must be a real, unique layer name.
|
||||
name = self._ple_offload_name
|
||||
if name == "<unbound>":
|
||||
raise RuntimeError(
|
||||
"PLE pull: setup_cross_process_offload on a layer with no "
|
||||
"_ple_offload_name; the op would resolve nothing by name"
|
||||
)
|
||||
prior = _PULL_LAYERS.get(name)
|
||||
if prior is not None and prior is not self:
|
||||
raise RuntimeError(
|
||||
f"PLE pull: two layers armed under the name {name!r}; the op "
|
||||
"would resolve the wrong layer's pointers"
|
||||
)
|
||||
_PULL_LAYERS[name] = self
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
*args: object,
|
||||
**kwargs: object,
|
||||
) -> torch.Tensor:
|
||||
"""Wait for an offloaded result or delegate to ``forward_impl``."""
|
||||
if self._is_cpu_offloaded:
|
||||
if self._host_output_ptr is not None:
|
||||
torch.ops.vllm.ple_offload_wait_copy(
|
||||
self._gpu_output_buffer,
|
||||
hidden_states,
|
||||
input_ids.shape[0],
|
||||
self._ple_offload_name,
|
||||
)
|
||||
else:
|
||||
torch.ops.vllm.ple_offload_wait(
|
||||
self._sem.flag_tensor,
|
||||
self._gpu_output_buffer,
|
||||
hidden_states,
|
||||
)
|
||||
return self._gpu_output_buffer[: input_ids.shape[0]]
|
||||
return self.forward_impl(hidden_states, input_ids, *args, **kwargs)
|
||||
|
||||
def release_offloaded_output(
|
||||
self,
|
||||
stream: torch.cuda.Stream | None = None,
|
||||
) -> None:
|
||||
"""Mark the cross-process output buffer reusable on ``stream``."""
|
||||
if self._is_cpu_offloaded:
|
||||
self._sem.reset(stream)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-launch staleness check for the pull transport (#53899 class)
|
||||
# ---------------------------------------------------------------------------
|
||||
# WHAT IT TESTS. The transport's correctness rests on one precondition: when a
|
||||
# real request is published, every layer's semaphore must be CLEAR, so the
|
||||
# ``cuStreamWaitValue32(flag == DONE_VALUE)`` inside the forward (line 279 for
|
||||
# the pull path, line 215 for the IPC push path) can only be satisfied by the
|
||||
# copy THIS step publishes. If a leftover ``DONE_VALUE`` survives into a real
|
||||
# step, that wait is satisfied immediately by the PREVIOUS step's signal and
|
||||
# the forward consumes the previous step's PLE rows. That is vllm#53899 on
|
||||
# GB10/sm_121 exactly: "every forward consumes the previous step's PLE
|
||||
# outputs", for the life of the server, with ``cudagraph_mode=NONE`` correct.
|
||||
#
|
||||
# In this tree the precondition is held structurally -- the dummy signal is
|
||||
# consumed by the dummy forward, ``_release_outputs_impl`` resets every layer
|
||||
# and synchronizes the model stream while ``_after_dummy`` is set, and the
|
||||
# dummy->real transition takes a TP barrier -- but nothing ever READ the flag
|
||||
# back, so the structural argument had no instrument. This is that instrument.
|
||||
#
|
||||
# WHY NOT AN EPOCH WORD. The stronger fix is a monotone epoch: the launch
|
||||
# writes ``epoch = step``, the worker signals ``epoch``, and the placeholder
|
||||
# waits ``EQ(epoch)``, so a stale signal carries the previous epoch and cannot
|
||||
# satisfy the wait. It is not implementable on this transport as it stands.
|
||||
# The compare value is an IMMEDIATE OPERAND of ``cuStreamWaitValue32``
|
||||
# (``_ple_offload_wait_copy_impl``, this file, line 279), and that call is
|
||||
# issued inside the model forward, which is captured: PIECEWISE and
|
||||
# FULL_AND_PIECEWISE both bake the wait into the graph with the value frozen at
|
||||
# capture time. Making the compare value change per replay needs the captured
|
||||
# node's parameters rewritten before each launch
|
||||
# (``cuGraphExecBatchMemOpNodeSetParams``); nothing in this tree's capture path
|
||||
# exposes the graph exec handle to the connector, and the wait is reached
|
||||
# through a torch custom op inside a compiled region. An epoch word would
|
||||
# therefore work only with ``cudagraph_mode=NONE``, i.e. only in the
|
||||
# configuration where #53899 does not reproduce. So: binary flag, read back.
|
||||
#
|
||||
# The check is HOST-SIDE and needs a stream synchronize to be ordered against
|
||||
# a pending reset, so it is never run inside a capture and is off by default.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def flag_check_is_authoritative(tp_rank: int) -> bool:
|
||||
"""Whether ``flag == RESET_VALUE`` before a launch is an INVARIANT here.
|
||||
|
||||
Only on the publishing rank (TP0). Why: the offload worker fans a
|
||||
result out to every registered TP target, each gated only on that target's
|
||||
own flag having been reset, so a non-publishing rank's flag can legitimately
|
||||
carry THIS step's signal before that rank reaches its own pre-launch check.
|
||||
Reading a set flag there and calling it stale terminates a healthy run.
|
||||
|
||||
On TP0 the invariant holds by construction: the worker learns of a request
|
||||
only from ``_launch``, which runs after the check, and the progress monitor
|
||||
refuses overlapping serving submissions, so the previous transaction has
|
||||
been released and (after a stream synchronize) its reset has retired.
|
||||
|
||||
A non-publishing rank cannot tell "this step's fan-out arrived early" from
|
||||
"last step's signal was never consumed", because the flag is BINARY and
|
||||
carries no transaction identity -- the same representational limit that
|
||||
rules out an epoch word. So those ranks are skipped, not checked weakly.
|
||||
"""
|
||||
return tp_rank == 0
|
||||
|
||||
|
||||
def stale_offload_flags(
|
||||
layers: "dict[str, PleOffloadLayer]",
|
||||
) -> list[tuple[str, int]]:
|
||||
"""Return ``(layer_name, flag_value)`` for every layer whose flag is set.
|
||||
|
||||
An empty list is the precondition holding. A non-empty list means at least
|
||||
one semaphore still carries a signal that no forward consumed, so the next
|
||||
``wait`` would be satisfied by the previous step's copy.
|
||||
|
||||
Pure and side-effect free: the caller owns the ordering (synchronize the
|
||||
model stream first, and never call this inside a stream capture). Layers
|
||||
that are not cross-process offloaded have no semaphore and are skipped.
|
||||
"""
|
||||
stale: list[tuple[str, int]] = []
|
||||
for name, layer in layers.items():
|
||||
if not getattr(layer, "_is_cpu_offloaded", False):
|
||||
continue
|
||||
value = int(layer._sem.flag_tensor.reshape(-1)[0].item())
|
||||
if value != CpuGpuSemaphore.RESET_VALUE:
|
||||
stale.append((name, value))
|
||||
return stale
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
@@ -31,8 +32,26 @@ from vllm.model_executor.layers.utils import dispatch_unquantized_gemm
|
||||
from vllm.model_executor.parameter import BasevLLMParameter
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
PIN_MEMORY,
|
||||
get_accelerator_view_from_cpu_tensor,
|
||||
)
|
||||
|
||||
DEFAULT_VOCAB_PADDING_SIZE = 64
|
||||
HOST_EMBED_TABLE_ENV = "VLLM_HOST_EMBED_TABLE"
|
||||
|
||||
|
||||
def host_embed_table_enabled() -> bool:
|
||||
"""Return whether the exact host-embedding opt-in is set."""
|
||||
return os.environ.get(HOST_EMBED_TABLE_ENV, "") == "1"
|
||||
|
||||
|
||||
def should_use_host_embed_table() -> bool:
|
||||
"""Keep the PLE offload worker's model construction on the meta device."""
|
||||
return (
|
||||
host_embed_table_enabled()
|
||||
and torch.empty(0).device.type != "meta"
|
||||
)
|
||||
|
||||
|
||||
class UnquantizedEmbeddingMethod(QuantizeMethodBase):
|
||||
@@ -91,6 +110,46 @@ class UnquantizedEmbeddingMethod(QuantizeMethodBase):
|
||||
return layer
|
||||
|
||||
|
||||
class _PinnedHostUnquantizedEmbeddingMethod(UnquantizedEmbeddingMethod):
|
||||
"""Allocate an unquantized embedding shard without a CUDA transient."""
|
||||
|
||||
requires_device_loading = False
|
||||
|
||||
def __init__(self, *, device: torch.device, pin_memory: bool) -> None:
|
||||
self.device = device
|
||||
self.pin_memory = pin_memory
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
del input_size, output_size
|
||||
weight = Parameter(
|
||||
torch.empty(
|
||||
sum(output_partition_sizes),
|
||||
input_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
|
||||
layer.register_parameter("weight", weight)
|
||||
set_weight_attrs(weight, extra_weight_attrs)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# The serving layer must retain the same pinned allocation whose mapped
|
||||
# address was captured. CPU-only reference tests need no GEMM dispatch.
|
||||
del layer
|
||||
|
||||
|
||||
def pad_vocab_size(vocab_size: int, pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int:
|
||||
"""Pad the vocab size to the given value."""
|
||||
return ((vocab_size + pad_to - 1) // pad_to) * pad_to
|
||||
@@ -572,6 +631,124 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
return s
|
||||
|
||||
|
||||
class PinnedHostVocabParallelEmbedding(VocabParallelEmbedding):
|
||||
"""TP-sharded embedding read on CUDA through mapped pinned host memory.
|
||||
|
||||
Host provisioning must use measured pinned allocation, not tensor payload.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_embeddings: int,
|
||||
embedding_dim: int,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
org_num_embeddings: int | None = None,
|
||||
padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
|
||||
prefix: str = "",
|
||||
*,
|
||||
disable_tp: bool = False,
|
||||
create_accelerator_view: bool = True,
|
||||
) -> None:
|
||||
construction_device = torch.empty(0).device
|
||||
is_meta = construction_device.type == "meta"
|
||||
if (
|
||||
create_accelerator_view
|
||||
and not is_meta
|
||||
and current_platform.is_cuda()
|
||||
and not PIN_MEMORY
|
||||
):
|
||||
raise RuntimeError(
|
||||
"VLLM_HOST_EMBED_TABLE requires available pinned host memory"
|
||||
)
|
||||
|
||||
storage_device = construction_device if is_meta else torch.device("cpu")
|
||||
pin_memory = (
|
||||
not is_meta
|
||||
and current_platform.is_cuda()
|
||||
and PIN_MEMORY
|
||||
)
|
||||
quant_method = _PinnedHostUnquantizedEmbeddingMethod(
|
||||
device=storage_device,
|
||||
pin_memory=pin_memory,
|
||||
)
|
||||
super().__init__(
|
||||
num_embeddings,
|
||||
embedding_dim,
|
||||
params_dtype=params_dtype,
|
||||
org_num_embeddings=org_num_embeddings,
|
||||
padding_size=padding_size,
|
||||
prefix=prefix,
|
||||
disable_tp=disable_tp,
|
||||
quant_method=quant_method,
|
||||
)
|
||||
|
||||
# The custom fused kernel is specified for ordinary device-resident
|
||||
# shards, not device-mapped host allocations. Use the regular masked
|
||||
# embedding path whose CUDA tensor argument is the mapped view.
|
||||
self.use_fused_embedding = False
|
||||
self._uva_weight: torch.Tensor | None = None
|
||||
if (
|
||||
create_accelerator_view
|
||||
and self.weight.device.type == "cpu"
|
||||
and current_platform.is_cuda()
|
||||
):
|
||||
if not self.weight.is_pinned():
|
||||
raise RuntimeError(
|
||||
"VLLM_HOST_EMBED_TABLE weight was not allocated pinned"
|
||||
)
|
||||
self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight)
|
||||
|
||||
@property
|
||||
def accelerator_weight_view(self) -> torch.Tensor | None:
|
||||
return self._uva_weight
|
||||
|
||||
def _apply(self, fn, recurse: bool = True):
|
||||
# Probe the transformation without applying it to the pinned shard.
|
||||
# Device-only parent moves remain no-ops, but silently retaining this
|
||||
# layer's dtype during a parent dtype conversion would create a
|
||||
# mixed-dtype model.
|
||||
del recurse
|
||||
probe = torch.empty(0, dtype=self.weight.dtype, device=self.weight.device)
|
||||
if fn(probe).dtype != self.weight.dtype:
|
||||
raise RuntimeError("Pinned host embedding has a fixed dtype")
|
||||
return self
|
||||
|
||||
def _lookup_weight(self, input_: torch.Tensor) -> torch.Tensor:
|
||||
if input_.device.type == "cpu":
|
||||
return self.weight
|
||||
if self._uva_weight is None:
|
||||
raise RuntimeError(
|
||||
"Host embedding received accelerator IDs without a UVA view"
|
||||
)
|
||||
if self._uva_weight.device != input_.device:
|
||||
raise RuntimeError(
|
||||
"Host embedding UVA view and input IDs are on different devices"
|
||||
)
|
||||
return self._uva_weight
|
||||
|
||||
def forward(self, input_: torch.Tensor) -> torch.Tensor:
|
||||
if self.tp_size == 1:
|
||||
return F.embedding(
|
||||
input_.long(),
|
||||
self._lookup_weight(input_),
|
||||
)
|
||||
|
||||
masked_input, input_mask = get_masked_input_and_mask(
|
||||
input_,
|
||||
self.shard_indices.org_vocab_start_index,
|
||||
self.shard_indices.org_vocab_end_index,
|
||||
self.shard_indices.num_org_vocab_padding,
|
||||
self.shard_indices.added_vocab_start_index,
|
||||
self.shard_indices.added_vocab_end_index,
|
||||
)
|
||||
output_parallel = F.embedding(
|
||||
masked_input.long(),
|
||||
self._lookup_weight(input_),
|
||||
)
|
||||
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||
return tensor_model_parallel_all_reduce(output_parallel)
|
||||
|
||||
|
||||
# --8<-- [start:parallel_lm_head]
|
||||
@PluggableLayer.register("parallel_lm_head")
|
||||
class ParallelLMHead(VocabParallelEmbedding):
|
||||
|
||||
@@ -883,12 +883,25 @@ class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConf
|
||||
)
|
||||
# Checked again in Qwen4ExpModelState; rejecting it here keeps the
|
||||
# engine from loading weights first.
|
||||
# vllm#54709: PLE only needs raw input_ids on the rank that owns a PLE
|
||||
# layer. Allow PP>1 as long as every PLE layer lives on pipeline rank 0.
|
||||
if text_config.ple_layer_ids and parallel_config.pipeline_parallel_size > 1:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp N-gram PLE embedding requires pipeline_parallel_size=1 "
|
||||
"because non-first pipeline ranks do not receive the raw input_ids "
|
||||
"it needs. Please run with PP=1."
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
|
||||
rank0_start, rank0_end = get_pp_indices(
|
||||
int(text_config.num_hidden_layers),
|
||||
0,
|
||||
parallel_config.pipeline_parallel_size,
|
||||
)
|
||||
ple_idx = [int(i) - 1 for i in text_config.ple_layer_ids]
|
||||
if min(ple_idx) < rank0_start or max(ple_idx) >= rank0_end:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp N-gram PLE embedding requires every PLE layer to "
|
||||
f"live on pipeline rank 0 (layers {rank0_start}.."
|
||||
f"{rank0_end - 1}), but PLE layers are at {ple_idx}. Give "
|
||||
"rank 0 more layers via VLLM_PP_LAYER_PARTITION, or run "
|
||||
"with PP=1."
|
||||
)
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
if multimodal_config is not None and multimodal_config.language_model_only:
|
||||
_strip_qwen4_exp_mrope(vllm_config.model_config)
|
||||
|
||||
@@ -119,6 +119,15 @@ class QSAIndexer(nn.Module):
|
||||
eps=float(getattr(config, "rms_norm_eps", 1e-6)),
|
||||
)
|
||||
|
||||
# QSAKeyStateCache rows now carry
|
||||
# a tag word the NVIDIA pre-indexer writes and validates; this path
|
||||
# neither writes nor checks it, so it would run the widened layout
|
||||
# with no ring validation at all. Refuse it rather than share it.
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA on the AMD path is not supported on this fork: the "
|
||||
"ring-row tag contract is implemented for the NVIDIA "
|
||||
"pre-indexer only (vllm/models/qwen4_exp/nvidia/ops/qsa_pre_indexer.py)."
|
||||
)
|
||||
cache_config = vllm_config.cache_config
|
||||
cache_prefix = f"{prefix}." if prefix else ""
|
||||
self.raw_key_cache = QSAKeyStateCache(
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.transformers_utils.configs.qwen4_exp import Qwen4ExpTextConfig
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
|
||||
def make_mtp_hidden_buffer(
|
||||
|
||||
@@ -15,6 +15,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.distributed import get_dp_group, get_etp_group, get_tp_group
|
||||
from vllm.forward_context import DPMetadata, get_forward_context
|
||||
@@ -422,6 +423,13 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
max_total_tokens: int = 0,
|
||||
data_parallel_rank: int = 0,
|
||||
) -> None:
|
||||
# Fork: never build the UVA table beside the E2 pull transport --
|
||||
# that would be a second offload of the same PLE table.
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
raise RuntimeError(
|
||||
"Qwen4ExpPLEPinnedHostEmbedding (upstream UVA PLE backend) "
|
||||
"refused: VLLM_E2_PLE_PULL_TRANSPORT=1 owns the PLE table"
|
||||
)
|
||||
if not is_uva_available():
|
||||
raise RuntimeError("Engram CPU offload requires UVA support")
|
||||
super().__init__(
|
||||
|
||||
@@ -44,6 +44,42 @@ from vllm.v1.kv_cache_interface import (
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
|
||||
# Ring-row tag contract. Every raw-key ring row is the key (bf16), padded to a
|
||||
# whole int64, followed by a tail of four int64 words:
|
||||
# (pos_t, pos_h, pos_w, tag), tag = logical_position + 1.
|
||||
# The tail exists on every ring, with or without cached RoPE positions: the tag
|
||||
# is what lets the reader prove the row it is about to pool was committed by
|
||||
# this request for exactly that logical position. 0 is never a valid tag (so a
|
||||
# cleared row can never validate) and the manager clears the tag word of every
|
||||
# newly claimed ring block before its new owner's first step.
|
||||
QSA_RING_TAIL_WORDS = 4
|
||||
QSA_RING_TAG_WORD = 3
|
||||
QSA_RING_BF16_PER_INT64 = 4
|
||||
QSA_RING_TAIL_ELEMENTS = QSA_RING_TAIL_WORDS * QSA_RING_BF16_PER_INT64
|
||||
|
||||
|
||||
def qsa_ring_row_tags(logical_positions: torch.Tensor) -> torch.Tensor:
|
||||
"""The tag the writer stores for each committed row (int64)."""
|
||||
return logical_positions.to(torch.int64) + 1
|
||||
|
||||
|
||||
def qsa_ring_group_rows_valid(
|
||||
found_tags: torch.Tensor,
|
||||
source_positions: torch.Tensor,
|
||||
historical: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Reference for the pre-indexer's per-group check (torch, any device).
|
||||
|
||||
``found_tags``/``source_positions``/``historical`` are ``[groups, ratio]``:
|
||||
the tag word read from each source row's ring slot, the row's logical
|
||||
position, and whether the row must come from the ring (it precedes the
|
||||
chunk start). Returns a ``[groups]`` bool: True when EVERY historical row
|
||||
carries ``source_position + 1``. In-chunk rows are never checked.
|
||||
"""
|
||||
expected = qsa_ring_row_tags(source_positions)
|
||||
bad = historical & (found_tags.to(torch.int64) != expected)
|
||||
return ~bad.any(dim=-1)
|
||||
|
||||
|
||||
def canonical_qsa_rope_positions(positions: torch.Tensor) -> torch.Tensor:
|
||||
"""Return exact per-token positions as ``[tokens, 1, 3]`` int64 rows."""
|
||||
@@ -868,37 +904,65 @@ class _QSAStateCache(nn.Module, AttentionLayerBase):
|
||||
|
||||
|
||||
class QSAKeyStateCache(_QSAStateCache):
|
||||
"""Raw BF16 key, optionally followed by exact int64 MRoPE positions."""
|
||||
"""Raw BF16 key followed by the int64 ring tail (pos_t, pos_h, pos_w, tag).
|
||||
|
||||
_BF16_PER_INT64 = 4
|
||||
The position words are consumed only when ``cache_rope_positions``; the tag
|
||||
word is always written and always validated."""
|
||||
|
||||
_BF16_PER_INT64 = QSA_RING_BF16_PER_INT64
|
||||
_NUM_ROPE_AXES = 3
|
||||
|
||||
def __init__(self, *, cache_rope_positions: bool = False, **kwargs) -> None:
|
||||
key_head_size = int(kwargs.pop("head_size"))
|
||||
self.key_head_size = key_head_size
|
||||
self.cache_rope_positions = bool(cache_rope_positions)
|
||||
# Element offset of the int64 tail (the name predates the tag word).
|
||||
self.rope_position_offset = (
|
||||
(key_head_size + self._BF16_PER_INT64 - 1) // self._BF16_PER_INT64
|
||||
) * self._BF16_PER_INT64
|
||||
storage_head_size = key_head_size
|
||||
if self.cache_rope_positions:
|
||||
storage_head_size = self.rope_position_offset + (
|
||||
self._NUM_ROPE_AXES * self._BF16_PER_INT64
|
||||
)
|
||||
storage_head_size = self.rope_position_offset + QSA_RING_TAIL_ELEMENTS
|
||||
super().__init__(head_size=storage_head_size, **kwargs)
|
||||
self._ring_tag_cache: torch.Tensor | None = None
|
||||
self._ring_tail: torch.Tensor | None = None
|
||||
|
||||
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||
super().bind_kv_cache(kv_cache)
|
||||
qsa_cache = self.kv_cache
|
||||
tail_end = self.rope_position_offset + QSA_RING_TAIL_ELEMENTS
|
||||
tail = qsa_cache[..., self.rope_position_offset : tail_end].view(torch.int64)
|
||||
assert tail.shape[-1] == QSA_RING_TAIL_WORDS
|
||||
# Derived on access so `kv_cache` stays the only reference to the bound
|
||||
# storage: clearing it (e.g. after CUDA graph memory profiling) must free
|
||||
# the cache, or the freed block stays pinned in the allocator. Only the
|
||||
# tag word is read outside a bind, so it is the one cached attribute.
|
||||
self._ring_tail = tail
|
||||
self._ring_tag_cache = tail[..., QSA_RING_TAG_WORD : QSA_RING_TAG_WORD + 1]
|
||||
|
||||
# Derived on access so `kv_cache` stays the only reference to the bound
|
||||
# storage: clearing it (e.g. after CUDA graph memory profiling) must free
|
||||
# the cache, or the freed block stays pinned in the allocator.
|
||||
@property
|
||||
def key_cache(self) -> torch.Tensor:
|
||||
return self.kv_cache[..., : self.key_head_size]
|
||||
|
||||
@property
|
||||
def rope_position_cache(self) -> torch.Tensor | None:
|
||||
if not self.cache_rope_positions:
|
||||
if not self.cache_rope_positions or self._ring_tail is None:
|
||||
return None
|
||||
return self.kv_cache[..., self.rope_position_offset :].view(torch.int64)
|
||||
return self._ring_tail[..., : self._NUM_ROPE_AXES]
|
||||
|
||||
@property
|
||||
def ring_tag_cache(self) -> torch.Tensor | None:
|
||||
return self._ring_tag_cache
|
||||
|
||||
def clear_ring_row_tags(self, block_ids: torch.Tensor) -> None:
|
||||
"""Zero the tag word of every row of ``block_ids`` (ring-row tag claim clear).
|
||||
|
||||
Only the tag word: a claimed ring block's keys and positions stay as
|
||||
the previous occupant left them, and the zero tag is what makes every
|
||||
such row fail validation until this request commits it."""
|
||||
if self._ring_tag_cache is None:
|
||||
raise RuntimeError(
|
||||
f"{self.prefix}: ring tag clear before the KV cache was bound"
|
||||
)
|
||||
self._ring_tag_cache.index_fill_(0, block_ids, 0)
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
# Hold the open group's committed keys plus every row a speculative
|
||||
@@ -936,6 +1000,9 @@ class QSACompressedKeyCache(_QSAStateCache):
|
||||
|
||||
|
||||
__all__ = [
|
||||
"QSA_RING_TAG_WORD",
|
||||
"QSA_RING_TAIL_ELEMENTS",
|
||||
"QSA_RING_TAIL_WORDS",
|
||||
"QSACompressedKeyCache",
|
||||
"QSAForwardMetadata",
|
||||
"QSAKeyStateCache",
|
||||
@@ -944,4 +1011,6 @@ __all__ = [
|
||||
"canonical_qsa_rope_positions",
|
||||
"circular_qsa_slot_mapping",
|
||||
"compressed_qsa_slot_mapping",
|
||||
"qsa_ring_group_rows_valid",
|
||||
"qsa_ring_row_tags",
|
||||
]
|
||||
|
||||
@@ -20,6 +20,7 @@ from ..common.qsa_cache import (
|
||||
QSAForwardMetadata,
|
||||
QSAKeyStateCache,
|
||||
canonical_qsa_rope_positions,
|
||||
qsa_ring_row_tags,
|
||||
)
|
||||
from .ops.qsa_prepare import qsa_prepare
|
||||
|
||||
@@ -339,6 +340,7 @@ class QSAIndexer(nn.Module):
|
||||
main_slot_mapping=slot_mapping[:num_tokens],
|
||||
main_k_scale=attn._k_scale_float,
|
||||
main_v_scale=attn._v_scale_float,
|
||||
layer_id=self.layer_id,
|
||||
)
|
||||
else:
|
||||
# Unfused reference path
|
||||
@@ -405,6 +407,18 @@ class QSAIndexer(nn.Module):
|
||||
raw_metadata.slot_mapping,
|
||||
position_rows,
|
||||
)
|
||||
# The unfused writer commits the same tag word as the fused
|
||||
# one, so both paths leave byte-identical rings. (This reference
|
||||
# reader does not validate tags; the fused kernel does.)
|
||||
ring_tag_cache = getattr(raw_key_state_cache, "ring_tag_cache", None)
|
||||
if ring_tag_cache is not None:
|
||||
qsa_store_cache_rows(
|
||||
ring_tag_cache,
|
||||
raw_metadata.slot_mapping,
|
||||
qsa_ring_row_tags(raw_metadata.logical_positions[:num_tokens]).view(
|
||||
-1, 1, 1
|
||||
),
|
||||
)
|
||||
|
||||
if self.skip_topk:
|
||||
if out is None:
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Qwen4Exp model."""
|
||||
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
@@ -9,8 +10,10 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
@@ -29,7 +32,9 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
PinnedHostVocabParallelEmbedding,
|
||||
VocabParallelEmbedding,
|
||||
should_use_host_embed_table,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
HasInnerState,
|
||||
@@ -59,6 +64,7 @@ from vllm.model_executor.models.qwen3_vl import (
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
StageMissingLayer,
|
||||
WeightsMapper,
|
||||
_merge_multimodal_embeddings,
|
||||
@@ -73,6 +79,11 @@ from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.e2_guards import (
|
||||
assert_compiled_submodule,
|
||||
assert_graph_profile_is_executable,
|
||||
assert_ple_mode,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
from ..common.mtp import make_mtp_hidden_buffer
|
||||
@@ -80,7 +91,9 @@ from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup
|
||||
from .ple_layer import Qwen4ExpPLELayer
|
||||
from .qsa import Qwen4ExpQSAAttention
|
||||
from .qsa import Qwen4ExpQSAAttention, load_qsa_static_kv_scales
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Transformers v5.18 renamed `qwen_sparse_attention` to `indexed_attention`
|
||||
# TODO: Delete qwen_... once Transformers 5.18.0 is the minimum required version.
|
||||
@@ -161,6 +174,104 @@ _EXTRA_WEIGHTS_MAPPER = WeightsMapper(
|
||||
)
|
||||
|
||||
|
||||
_QSA_KV_SCALES_ENV = "VLLM_QSA_KV_SCALES"
|
||||
|
||||
|
||||
def _maybe_load_qsa_static_kv_scales(model: nn.Module) -> None:
|
||||
"""Apply a static QSA K/V scale sidecar when VLLM_QSA_KV_SCALES names one.
|
||||
|
||||
Placed where the loader's own docstring requires: after checkpoint weights
|
||||
are loaded, before profiling, warmup or cudagraph capture. Unset env var ->
|
||||
no-op (the attention layer logs the default 1.0 per layer so the
|
||||
uncalibrated state stays visible). Set but unusable -> the exception
|
||||
propagates: a calibration that was requested and silently not applied is
|
||||
worse than one never requested.
|
||||
"""
|
||||
path = os.environ.get(_QSA_KV_SCALES_ENV, "").strip()
|
||||
if not path:
|
||||
return
|
||||
# The PLE offload worker builds the whole target on the meta device and loads
|
||||
# only PLE weights through load_weights; a fill_ on its meta scale buffers
|
||||
# would log success without calibrating any GPU worker.
|
||||
local_qsa: list[Qwen4ExpQSAAttention] = []
|
||||
inner: Qwen4ExpModel | None = None
|
||||
for module in model.modules():
|
||||
if isinstance(module, Qwen4ExpModel) and inner is None:
|
||||
inner = module
|
||||
if isinstance(module, Qwen4ExpQSAAttention):
|
||||
local_qsa.append(module)
|
||||
for layer in local_qsa:
|
||||
k_scale = getattr(layer, "_k_scale", None)
|
||||
if isinstance(k_scale, torch.Tensor) and k_scale.device.type == "meta":
|
||||
logger.info(
|
||||
"Qwen4Exp QSA: model is on meta (PLE offload worker); "
|
||||
"static K/V scales are applied by the GPU workers only"
|
||||
)
|
||||
return
|
||||
if inner is None:
|
||||
raise ValueError("Qwen4Exp QSA: no Qwen4ExpModel found for the sidecar")
|
||||
if not inner._qsa_layer_ids:
|
||||
# A requested calibration for a model without QSA layers is a
|
||||
# configuration error, never a silent no-op.
|
||||
raise ValueError(
|
||||
f"Qwen4Exp QSA: {_QSA_KV_SCALES_ENV} is set but the model has no "
|
||||
"QSA (indexer) attention layers"
|
||||
)
|
||||
# Whole-model QSA layer set (all pipeline stages) = "<stack>.<idx>.self_attn.attn"
|
||||
# for every configured QSA index. The stack prefix is read from a LOCAL QSA
|
||||
# layer's registered layer_name whenever one exists: this hook also runs
|
||||
# inside the nested Qwen4ExpForCausalLM.load_weights, where a module path
|
||||
# relative to that submodule ("model.layers") would not match the names
|
||||
# registered under the outer prefix ("language_model.model.layers") - the
|
||||
# earlier boot failure. Only a stage with no QSA layer
|
||||
# falls back to the module path (then no local name can disagree).
|
||||
if local_qsa:
|
||||
stack, sep, tail = local_qsa[0].layer_name.rpartition(".layers.")
|
||||
if not sep or not tail.split(".")[0].isdigit():
|
||||
raise ValueError(
|
||||
f"Qwen4Exp QSA: unexpected layer name {local_qsa[0].layer_name!r}"
|
||||
)
|
||||
layers_path = f"{stack}.layers"
|
||||
else:
|
||||
layers_path = next(
|
||||
(name for name, module in model.named_modules() if module is inner.layers),
|
||||
None,
|
||||
)
|
||||
if layers_path is None:
|
||||
raise ValueError("Qwen4Exp QSA: cannot locate the decoder layer stack")
|
||||
expected = {
|
||||
f"{layers_path}.{idx}.self_attn.attn" for idx in inner._qsa_layer_ids
|
||||
}
|
||||
if not local_qsa:
|
||||
# A pipeline stage without a full-attention layer (only for a very
|
||||
# uneven split): validate the complete sidecar here too, apply nothing.
|
||||
logger.warning(
|
||||
"Qwen4Exp QSA: this pipeline stage holds no QSA layer "
|
||||
"(pipeline layers %d-%d); sidecar %s validated, nothing to apply",
|
||||
inner.start_layer,
|
||||
inner.end_layer,
|
||||
path,
|
||||
)
|
||||
# strict: every QSA layer of the whole model must be in the sidecar. An empty
|
||||
# or partial file silently leaving layers at scale 1.0 is exactly the failure
|
||||
# this call site exists to prevent.
|
||||
applied = load_qsa_static_kv_scales(
|
||||
model, path, strict=True, expected_layer_names=expected
|
||||
)
|
||||
shown = ", ".join(sorted(applied)[:4])
|
||||
logger.info(
|
||||
"Qwen4Exp QSA: applied static K/V scales from %s to %d of %d layer(s) "
|
||||
"(pipeline layers %d-%d): %s%s",
|
||||
path,
|
||||
len(applied),
|
||||
len(expected),
|
||||
inner.start_layer,
|
||||
inner.end_layer,
|
||||
shown,
|
||||
" ..." if len(applied) > 4 else "",
|
||||
)
|
||||
|
||||
|
||||
class Qwen4ExpSparseMoeBlock(Qwen3NextSparseMoeBlock):
|
||||
"""Qwen3Next MoE with Qwen4Exp HC validation."""
|
||||
|
||||
@@ -366,6 +477,25 @@ class Qwen4ExpMixtureOfExperts(MixtureOfExperts):
|
||||
moe.experts.update_expert_map()
|
||||
|
||||
|
||||
# Compile-profile guard: torch.compile restored on the NVIDIA
|
||||
# path, reverting #55272's removal. Every request-dependent region is a custom
|
||||
# op listed in CompilationConfig._attention_ops (qsa_with_output,
|
||||
# ple_short_conv, compute_ple_ngram_ids, ple_{start,finalize}_prefetch), so
|
||||
# VLLM_COMPILE + FULL_AND_PIECEWISE is executable again with breakable capture
|
||||
# off. query_start_loc / ngram_context are fixed-capacity since #55272
|
||||
# (model_state.py); they stay marked dynamic as on v2.0.1 and only ever reach
|
||||
# the opaque PLE ops.
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
"query_start_loc": 0,
|
||||
"ngram_context": 0,
|
||||
"deepstack_input_embeds": 0,
|
||||
}
|
||||
)
|
||||
class Qwen4ExpModel(nn.Module):
|
||||
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
|
||||
|
||||
@@ -383,7 +513,31 @@ class Qwen4ExpModel(nn.Module):
|
||||
if layer_type in _QSA_LAYER_TYPES
|
||||
and getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
|
||||
self.uses_host_embed_table = (
|
||||
get_pp_group().is_first_rank
|
||||
and should_use_host_embed_table()
|
||||
)
|
||||
if self.uses_host_embed_table and getattr(
|
||||
config, "tie_word_embeddings", False
|
||||
):
|
||||
raise ValueError(
|
||||
"VLLM_HOST_EMBED_TABLE=1 is incompatible with "
|
||||
"tie_word_embeddings=True"
|
||||
)
|
||||
# PP ownership: the input embedding runs on
|
||||
# the first stage only (see forward); later stages carried an unused
|
||||
# 248320 x 2560 bf16 table (0.59 GiB per GPU at TP2) beside the MTP
|
||||
# drafter's own copy. PP=1 ownership is unchanged, and the PLE offload
|
||||
# worker retains meta construction.
|
||||
if get_pp_group().is_first_rank:
|
||||
embedding_cls = (
|
||||
PinnedHostVocabParallelEmbedding
|
||||
if self.uses_host_embed_table
|
||||
else VocabParallelEmbedding
|
||||
)
|
||||
self.embed_tokens = embedding_cls(self.vocab_size, config.hidden_size)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
def get_layer(prefix: str) -> Qwen4ExpDecoderLayer:
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
@@ -630,6 +784,13 @@ class Qwen4ExpForCausalLM(
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
# Compile-profile guard: the only qualified profile on this tree is VLLM_COMPILE +
|
||||
# FULL_AND_PIECEWISE with breakable capture off (the v2.0.1 identity).
|
||||
# Fail at construction, before profiling or capture, on anything else.
|
||||
# Qwen4ExpForConditionalGeneration constructs this class; Qwen4ExpMTP
|
||||
# does not, and guards itself.
|
||||
assert_graph_profile_is_executable(vllm_config, role="target")
|
||||
assert_ple_mode(vllm_config, role="target")
|
||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||
if vllm_config.lora_config is not None:
|
||||
# LoRA does not support the merged QKV/indexer projection, so its
|
||||
@@ -645,11 +806,19 @@ class Qwen4ExpForCausalLM(
|
||||
self.model = Qwen4ExpModel(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
# has_compiled_submodule() at gpu/cudagraph_utils.py:605 is what capture
|
||||
# actually tests; check the same property here.
|
||||
assert_compiled_submodule(self.model, vllm_config, role="target")
|
||||
# PP ownership: logits are computed on the
|
||||
# last stage only; earlier stages carried an unused 0.59 GiB head.
|
||||
if get_pp_group().is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.model.make_empty_intermediate_tensors
|
||||
@@ -869,7 +1038,9 @@ class Qwen4ExpForCausalLM(
|
||||
self,
|
||||
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
|
||||
)
|
||||
return loader.load_weights(weights, mapper=mapper)
|
||||
loaded = loader.load_weights(weights, mapper=mapper)
|
||||
_maybe_load_qsa_static_kv_scales(self)
|
||||
return loaded
|
||||
|
||||
|
||||
class Qwen4ExpProcessingInfo(Qwen3VLProcessingInfo):
|
||||
@@ -928,6 +1099,14 @@ class Qwen4ExpForConditionalGeneration(
|
||||
self.model_config = vllm_config.model_config
|
||||
self.multimodal_config = multimodal_config
|
||||
self.language_model_only = multimodal_config.language_model_only
|
||||
# PP ownership: the encoder cache and the image
|
||||
# embeddings exist on the first pipeline stage only; later stages
|
||||
# carried an unused vision tower (~0.42 GiB per GPU at TP2). Absence
|
||||
# on a later stage is an ownership decision, not language_model_only:
|
||||
# the multimodal configuration stays global.
|
||||
self._owns_vision_tower = (
|
||||
not self.language_model_only and get_pp_group().is_first_rank
|
||||
)
|
||||
if self.language_model_only:
|
||||
self.use_data_parallel = False
|
||||
self.is_multimodal_pruning_enabled = False
|
||||
@@ -936,6 +1115,12 @@ class Qwen4ExpForConditionalGeneration(
|
||||
self._tokenizer = None
|
||||
self.visual = StageMissingLayer("vision_tower")
|
||||
self._tower_model_names = []
|
||||
elif not self._owns_vision_tower:
|
||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||
self._init_video_pruning(multimodal_config)
|
||||
self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
|
||||
self.visual = StageMissingLayer("vision_tower")
|
||||
self._tower_model_names = []
|
||||
else:
|
||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||
self._init_video_pruning(multimodal_config)
|
||||
@@ -982,7 +1167,16 @@ class Qwen4ExpForConditionalGeneration(
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
if not get_pp_group().is_first_rank and self.use_deepstack:
|
||||
# Deepstack levels are injected into the first len(indexes) decoder
|
||||
# layers on the first stage and are not forwarded through PP; keep the
|
||||
# boundary assertion independent of use_deepstack (False on stages
|
||||
# without a tower) so a future checkpoint with deepstack cannot be
|
||||
# split across the boundary silently.
|
||||
if (
|
||||
not get_pp_group().is_first_rank
|
||||
and not self.language_model_only
|
||||
and bool(config.vision_config.deepstack_visual_indexes)
|
||||
):
|
||||
assert self.language_model.model.start_layer >= len(
|
||||
config.vision_config.deepstack_visual_indexes
|
||||
), (
|
||||
@@ -1073,7 +1267,9 @@ class Qwen4ExpForConditionalGeneration(
|
||||
self,
|
||||
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
|
||||
)
|
||||
return loader.load_weights(weights, mapper=mapper)
|
||||
loaded = loader.load_weights(weights, mapper=mapper)
|
||||
_maybe_load_qsa_static_kv_scales(self)
|
||||
return loaded
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_dtype_from_config(
|
||||
|
||||
@@ -38,13 +38,7 @@ class Qwen4ExpModelState(MambaHybridModelState):
|
||||
self.ngram_eos_token_id = 0
|
||||
return
|
||||
|
||||
if vllm_config.parallel_config.pipeline_parallel_size > 1:
|
||||
raise RuntimeError(
|
||||
"N-gram PLE embedding currently requires "
|
||||
"pipeline_parallel_size=1 because non-first pipeline ranks do "
|
||||
"not receive the raw input_ids required by PLE. Please run "
|
||||
"with PP=1."
|
||||
)
|
||||
# vllm#54709: PP gate moved to config.py (placement-based).
|
||||
|
||||
self.ngram_context_len = int(config.ngram_size) - 1
|
||||
if self.ngram_context_len <= 0:
|
||||
|
||||
@@ -20,6 +20,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig, replace, set_current_vllm_config
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
@@ -30,7 +31,9 @@ from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
PinnedHostVocabParallelEmbedding,
|
||||
VocabParallelEmbedding,
|
||||
should_use_host_embed_table,
|
||||
)
|
||||
from vllm.model_executor.model_loader.utils import configure_quant_config
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
@@ -49,6 +52,11 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.e2_guards import (
|
||||
assert_compiled_submodule,
|
||||
assert_graph_profile_is_executable,
|
||||
assert_ple_mode,
|
||||
)
|
||||
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
@@ -165,6 +173,18 @@ def _make_draft_vllm_config(
|
||||
return draft_vllm_config
|
||||
|
||||
|
||||
# Compile restored on the drafter too (reverts #55272 here), as
|
||||
# on v2.0.1: both classes wrapped; the outer one compiles, the inner is
|
||||
# inlined while Dynamo traces.
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
"hidden_states": 0,
|
||||
}
|
||||
)
|
||||
class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
|
||||
|
||||
@@ -183,7 +203,20 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
self.hidden_size = config.hidden_size
|
||||
self.hc_count = config.hc_count
|
||||
|
||||
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, self.hidden_size)
|
||||
self.uses_host_embed_table = should_use_host_embed_table()
|
||||
if self.uses_host_embed_table and getattr(
|
||||
config, "tie_word_embeddings", False
|
||||
):
|
||||
raise ValueError(
|
||||
"VLLM_HOST_EMBED_TABLE=1 is incompatible with "
|
||||
"tie_word_embeddings=True"
|
||||
)
|
||||
embedding_cls = (
|
||||
PinnedHostVocabParallelEmbedding
|
||||
if self.uses_host_embed_table
|
||||
else VocabParallelEmbedding
|
||||
)
|
||||
self.embed_tokens = embedding_cls(self.vocab_size, self.hidden_size)
|
||||
draft_vllm_config = _make_draft_vllm_config(
|
||||
vllm_config,
|
||||
self.mtp_start_layer_idx,
|
||||
@@ -288,7 +321,11 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
hidden_size = self.hidden_size
|
||||
prev_block_output: torch.Tensor | None = None
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
# Branch on the inputs, not the rank (vllm PR #46994's rule, applied
|
||||
# there to Qwen3_5MTP / DeepSeek MTP): the drafter is built entirely on
|
||||
# the LAST pipeline stage, where is_first_rank is False, and it always
|
||||
# starts from the target's hidden state + token ids it embeds itself.
|
||||
if intermediate_tensors is None:
|
||||
assert hidden_states is not None
|
||||
if inputs_embeds is None:
|
||||
assert input_ids is not None
|
||||
@@ -310,7 +347,6 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
hidden_states = hidden_states.flatten(-2)
|
||||
prev_block_output = inputs_embeds
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
@@ -377,6 +413,11 @@ class Qwen4ExpMTP(
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
# The drafter is NOT reached through Qwen4ExpForCausalLM --
|
||||
# this class builds Qwen4ExpMultiTokenPredictor directly, from a config
|
||||
# rebuilt at mtp.py:159 -- so its graph profile is checked here.
|
||||
assert_graph_profile_is_executable(vllm_config, role="draft")
|
||||
assert_ple_mode(vllm_config, role="draft")
|
||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||
self.vllm_config = vllm_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
@@ -387,6 +428,7 @@ class Qwen4ExpMTP(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "mtp"),
|
||||
)
|
||||
assert_compiled_submodule(self.model, vllm_config, role="draft")
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
if config.tie_word_embeddings:
|
||||
|
||||
@@ -10,12 +10,14 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
from ..common.ngram_embedding import (
|
||||
Qwen4ExpPLEDeviceEmbedding,
|
||||
@@ -176,6 +178,15 @@ class Qwen4ExpPLEFileGatherEmbedding(Qwen4ExpPLEEmbedding):
|
||||
self._host_rows[:num_tokens], non_blocking=True
|
||||
)
|
||||
|
||||
@property
|
||||
def prefetch_buffer(self) -> torch.Tensor:
|
||||
"""Side-stream lookup destination (declared mutated by the start op)."""
|
||||
return self._prefetch_buffer
|
||||
|
||||
def new_prefetch_output(self, num_tokens: int) -> torch.Tensor:
|
||||
"""Graph-owned output storage for the finalize op."""
|
||||
return self._prefetch_buffer.new_empty((num_tokens, self._output_dim))
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Return the rows staged for this step."""
|
||||
return self._staging[: hidden_states.shape[0]].flatten(-2)
|
||||
@@ -295,10 +306,16 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
*,
|
||||
data_parallel_rank: int,
|
||||
prefix: str,
|
||||
layer_name: str | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# Compile-profile guard: the owning Qwen4ExpPLELayer's static_forward_context key.
|
||||
# When set, request-dependent work runs through the custom ops at the
|
||||
# bottom of this file so VLLM_COMPILE never traces it (#55272 removed
|
||||
# them together with @support_torch_compile). None = direct calls.
|
||||
self.layer_name = layer_name
|
||||
self.embedding_dim = embedding_dim
|
||||
self.ngram_size = int(config.ngram_size)
|
||||
self.heads_per_ngram = int(config.heads_per_ngram)
|
||||
@@ -357,11 +374,18 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
if params_dtype is None:
|
||||
params_dtype = torch.get_default_dtype()
|
||||
engram_config = get_current_vllm_config().engram_config
|
||||
use_uva = engram_config is not None and engram_config.cpu_offload
|
||||
if use_uva and envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
# Fork: no silent fallback between transports.
|
||||
raise RuntimeError(
|
||||
"PLE backend selection: cpu_offload=True (upstream UVA) with "
|
||||
"VLLM_E2_PLE_PULL_TRANSPORT=1 -- refusing a double offload"
|
||||
)
|
||||
embedding_cls = (
|
||||
Qwen4ExpPLEFileGatherEmbedding
|
||||
if engram_config is not None and engram_config.host_file_gather
|
||||
else Qwen4ExpPLEPinnedHostEmbedding
|
||||
if engram_config is not None and engram_config.cpu_offload
|
||||
if use_uva
|
||||
else Qwen4ExpPLEDeviceEmbedding
|
||||
)
|
||||
self.ngram_embedding = embedding_cls(
|
||||
@@ -382,14 +406,21 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
max_total_tokens, self.ngram_heads, dtype=torch.long
|
||||
)
|
||||
weight = self.ngram_embedding.weight
|
||||
# Fork: backend= names the class (or, inside the pull transport's
|
||||
# offload process, the table it serves), so a smoke need not derive
|
||||
# the backend from pinned=. Fields before it are upstream's, unchanged.
|
||||
backend = type(self.ngram_embedding).__name__
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
backend = f"E2PullTransport.offload_table:{backend}"
|
||||
logger.info(
|
||||
"Initialized PLE embedding %s: quantization_method=%s, "
|
||||
"weight_dtype=%s, weight_device=%s, pinned=%s",
|
||||
"weight_dtype=%s, weight_device=%s, pinned=%s, backend=%s",
|
||||
embedding_prefix,
|
||||
type(embedding_quant_method).__name__,
|
||||
weight.dtype,
|
||||
weight.device,
|
||||
weight.is_pinned(),
|
||||
backend,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -402,6 +433,13 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
positions = torch.arange(seq_len, device=tokens.device, dtype=torch.int64)
|
||||
eos_positions = torch.where(tokens == eos_token_id, positions, -1)
|
||||
previous_eos_inclusive = torch.cummax(eos_positions, dim=1).values
|
||||
# Fork: never build the UVA table beside the E2 pull transport --
|
||||
# that would be a second offload of the same PLE table.
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
raise RuntimeError(
|
||||
"Qwen4ExpPLEPinnedHostEmbedding (upstream UVA PLE backend) "
|
||||
"refused: VLLM_E2_PLE_PULL_TRANSPORT=1 owns the PLE table"
|
||||
)
|
||||
previous_eos = torch.cat(
|
||||
[
|
||||
eos_positions.new_full((batch_size, 1), -1),
|
||||
@@ -509,12 +547,43 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
embedding = self.ngram_embedding
|
||||
if embedding.supports_prefetch or isinstance(
|
||||
embedding, Qwen4ExpPLEFileGatherEmbedding
|
||||
):
|
||||
if isinstance(embedding, Qwen4ExpPLEFileGatherEmbedding):
|
||||
# Fork: this base's third backend stages rows in prepare_inputs,
|
||||
# so it has no graph-owned lookup op.
|
||||
return embedding(hidden_states)
|
||||
ngram_ids = self.compute_ngram_ids(input_ids, query_start_loc, ngram_context)
|
||||
return self.ngram_embedding(ngram_ids).flatten(-2)
|
||||
if self.layer_name is None:
|
||||
if embedding.supports_prefetch:
|
||||
return embedding(hidden_states)
|
||||
ngram_ids = self.compute_ngram_ids(
|
||||
input_ids, query_start_loc, ngram_context
|
||||
)
|
||||
return self.ngram_embedding(ngram_ids).flatten(-2)
|
||||
|
||||
if embedding.supports_prefetch:
|
||||
output = embedding.new_prefetch_output(hidden_states.shape[0])
|
||||
torch.ops.vllm.qwen4_exp_ple_finalize_prefetch(output, self.layer_name)
|
||||
return output
|
||||
input_ids = input_ids.reshape(-1)
|
||||
# Request-layout-dependent ID generation stays outside PIECEWISE graphs
|
||||
# (a splitting op), as on v2.0.1; the op writes graph-owned storage.
|
||||
ngram_ids = input_ids.new_empty(
|
||||
(input_ids.numel(), self.ngram_heads), dtype=torch.long
|
||||
)
|
||||
torch.ops.vllm.qwen4_exp_compute_ple_ngram_ids(
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
ngram_ids,
|
||||
self.layer_name,
|
||||
)
|
||||
# The table lookup is opaque so Inductor never sees (or autotunes
|
||||
# against) the full PLE table -- #55272's ~50 GB activation peak.
|
||||
output = ngram_ids.new_empty(
|
||||
(ngram_ids.shape[0], self.ngram_heads * self.head_dim),
|
||||
dtype=embedding.weight.dtype,
|
||||
)
|
||||
torch.ops.vllm.qwen4_exp_ple_device_lookup(ngram_ids, output, self.layer_name)
|
||||
return output
|
||||
|
||||
def start_prefetch(
|
||||
self,
|
||||
@@ -527,6 +596,32 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
embedding = self.ngram_embedding
|
||||
if not embedding.supports_prefetch:
|
||||
return
|
||||
if self.layer_name is not None:
|
||||
# Forks onto the prefetch side stream; the join is in the finalize
|
||||
# op. Both are splitting ops, so neither end of the fork is ever
|
||||
# inside a piecewise capture.
|
||||
torch.ops.vllm.qwen4_exp_ple_start_prefetch(
|
||||
hidden_states,
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
embedding.prefetch_buffer,
|
||||
self.layer_name,
|
||||
)
|
||||
return
|
||||
self._start_prefetch_eager(
|
||||
hidden_states, input_ids, query_start_loc, ngram_context
|
||||
)
|
||||
|
||||
def _start_prefetch_eager(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> None:
|
||||
"""Compute IDs and launch the pinned lookup (runs outside compile)."""
|
||||
embedding = self.ngram_embedding
|
||||
ngram_ids = self.compute_ngram_ids(
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
@@ -624,6 +719,138 @@ def stage_checkpoint_rows(
|
||||
module.ngram_embedding.stage_rows(num_tokens)
|
||||
|
||||
|
||||
# --- compile support ------------------------------------------------
|
||||
# The four ops below are the NVIDIA PLE's boundary with torch.compile. Each
|
||||
# resolves its Qwen4ExpPLELayer through the forward context by name, as
|
||||
# v2.0.1's qwen4_exp_compute_ple_ngram_ids did before #55272 removed it.
|
||||
# Splitting ops (vllm/config/compilation.py ``_attention_ops``):
|
||||
# qwen4_exp_compute_ple_ngram_ids, qwen4_exp_ple_start_prefetch,
|
||||
# qwen4_exp_ple_finalize_prefetch.
|
||||
# qwen4_exp_ple_device_lookup is opaque but NOT a split: a gather plus the ETP
|
||||
# all-reduce is capturable; it only has to be hidden from Inductor.
|
||||
|
||||
|
||||
def _ple_embedding_of(layer_name: str) -> Qwen4ExpNGramEmbedding:
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
embedding = getattr(layer, "ple_embedding", None)
|
||||
if not isinstance(embedding, Qwen4ExpNGramEmbedding):
|
||||
raise TypeError(f"{layer_name} does not own a Qwen4Exp PLE embedding")
|
||||
if hasattr(type(embedding), "dequantize_pulled"):
|
||||
# Fork (pull transport + compile profile): the pull transport's GPU placeholder owns no
|
||||
# table and dispatches through PleOffloadLayer.forward; it must never
|
||||
# reach these device/UVA ops (ple_layer.py builds it without a
|
||||
# layer_name). Same marker ple_layer.py uses to select dequantisation.
|
||||
raise TypeError(
|
||||
f"{layer_name}: PLE pull transport module routed through an "
|
||||
"upstream device/UVA PLE op"
|
||||
)
|
||||
return embedding
|
||||
|
||||
|
||||
def qwen4_exp_compute_ple_ngram_ids(
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""Compute request-dependent PLE n-gram IDs outside piecewise graphs."""
|
||||
_ple_embedding_of(layer_name).compute_ngram_ids(
|
||||
input_ids, query_start_loc, ngram_context, output
|
||||
)
|
||||
|
||||
|
||||
def qwen4_exp_ple_device_lookup(
|
||||
ngram_ids: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""Look up device-resident PLE rows; keeps the table out of Inductor."""
|
||||
embedding = _ple_embedding_of(layer_name)
|
||||
output.copy_(embedding.ngram_embedding(ngram_ids).flatten(-2))
|
||||
|
||||
|
||||
def qwen4_exp_ple_start_prefetch(
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
prefetch_buffer: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""Start the pinned-host (UVA) lookup on the side stream."""
|
||||
embedding = _ple_embedding_of(layer_name)
|
||||
owned = embedding.ngram_embedding.prefetch_buffer
|
||||
if prefetch_buffer.data_ptr() != owned.data_ptr():
|
||||
raise RuntimeError(f"{layer_name}: PLE prefetch buffer identity changed")
|
||||
embedding._start_prefetch_eager(
|
||||
hidden_states, input_ids, query_start_loc, ngram_context
|
||||
)
|
||||
|
||||
|
||||
def qwen4_exp_ple_finalize_prefetch(output: torch.Tensor, layer_name: str) -> None:
|
||||
"""Join the side stream and write the pinned lookup into ``output``."""
|
||||
pinned = _ple_embedding_of(layer_name).ngram_embedding
|
||||
pinned._finalize_prefetch(pinned.prefetch_buffer, output)
|
||||
|
||||
|
||||
def _compute_ple_ngram_ids_fake(
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _ple_device_lookup_fake(
|
||||
ngram_ids: torch.Tensor, output: torch.Tensor, layer_name: str
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _ple_start_prefetch_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
prefetch_buffer: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _ple_finalize_prefetch_fake(output: torch.Tensor, layer_name: str) -> None:
|
||||
return None
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_compute_ple_ngram_ids",
|
||||
op_func=qwen4_exp_compute_ple_ngram_ids,
|
||||
mutates_args=["output"],
|
||||
fake_impl=_compute_ple_ngram_ids_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_ple_device_lookup",
|
||||
op_func=qwen4_exp_ple_device_lookup,
|
||||
mutates_args=["output"],
|
||||
fake_impl=_ple_device_lookup_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_ple_start_prefetch",
|
||||
op_func=qwen4_exp_ple_start_prefetch,
|
||||
mutates_args=["prefetch_buffer"],
|
||||
fake_impl=_ple_start_prefetch_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_ple_finalize_prefetch",
|
||||
op_func=qwen4_exp_ple_finalize_prefetch,
|
||||
mutates_args=["output"],
|
||||
fake_impl=_ple_finalize_prefetch_fake,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Qwen4ExpNGramEmbedding",
|
||||
]
|
||||
|
||||
@@ -26,13 +26,32 @@ def _is_sm120() -> bool:
|
||||
def _is_sm90() -> bool:
|
||||
"""True on sm_90 (H100/H200/H20): selects the sm_90 tuning table."""
|
||||
return current_platform.get_device_capability() == (9, 0)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _decode_e4m3_to_bf16(b):
|
||||
# Shifting the 7 magnitude bits left by 7 lands the E4M3 exponent in the
|
||||
# FP16 exponent field and the 3 mantissa bits at the top of the FP16
|
||||
# mantissa field, giving the right number under the wrong exponent bias
|
||||
# (E4M3 7 vs FP16 15). That is a constant factor of 2^8. E4M3 subnormals
|
||||
# land on FP16 subnormals and the same factor renormalizes them, which is
|
||||
# only true because sm_86 does not flush FP16 subnormals -- verified on
|
||||
# device, not assumed.
|
||||
#
|
||||
# Bit-exact against torch on all 254 finite codepoints: every E4M3 value
|
||||
# fits in BF16's 7 mantissa bits, and x256 is exact in FP16.
|
||||
#
|
||||
# Deviation: 0x7F/0xFF are NaN in e4m3fn but decode to +-480 here. The CUDA
|
||||
# writer clamps to +-448 and never emits them, so this is unreachable from
|
||||
# a cache written by reshape_and_cache_flash.
|
||||
b = b.to(tl.uint16)
|
||||
bits = ((b & 0x80) << 8) | ((b & 0x7F) << 7)
|
||||
return (bits.to(tl.uint16).to(tl.float16, bitcast=True) * 256.0).to(tl.bfloat16)
|
||||
@triton.jit(do_not_specialize=["num_rows", "num_requests"])
|
||||
def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||
q_ptr,
|
||||
k_cache_ptr,
|
||||
v_cache_ptr,
|
||||
k_scale_ptr,
|
||||
v_scale_ptr,
|
||||
indices_ptr,
|
||||
block_table_ptr,
|
||||
token_to_req_ptr,
|
||||
@@ -67,6 +86,7 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||
NUM_QUERY_HEADS: tl.constexpr,
|
||||
NUM_SPLITS: tl.constexpr,
|
||||
NUM_TILES: tl.constexpr,
|
||||
KV_DTYPE: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
IS_FP8: tl.constexpr,
|
||||
@@ -104,6 +124,10 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||
# here for the exp2-based online softmax.
|
||||
score_scale = softmax_scale * 1.4426950408889634
|
||||
|
||||
if KV_DTYPE != 0:
|
||||
k_scale_value = tl.load(k_scale_ptr)
|
||||
v_scale_value = tl.load(v_scale_ptr)
|
||||
|
||||
tile_end = tl.minimum(NUM_TILES, tl.cdiv(tl.minimum(valid_count, TOPK), BLOCK_N))
|
||||
|
||||
for tile in range(split_id, tile_end, NUM_SPLITS):
|
||||
@@ -132,44 +156,87 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||
valid &= (physical_page >= 0) & (physical_page < num_cache_blocks)
|
||||
# physical_page * block stride can overflow int32 for large caches.
|
||||
safe_page = tl.maximum(physical_page, 0).to(tl.int64)
|
||||
keys = tl.load(
|
||||
k_cache_ptr
|
||||
+ safe_page[None, :] * stride_k_block
|
||||
+ page_offset[None, :] * stride_k_token
|
||||
+ kv_head * stride_k_head
|
||||
+ dim_offsets[:, None],
|
||||
mask=valid[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
values = tl.load(
|
||||
v_cache_ptr
|
||||
+ safe_page[:, None] * stride_v_block
|
||||
+ page_offset[:, None] * stride_v_token
|
||||
+ kv_head * stride_v_head
|
||||
+ dim_offsets[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
if IS_FP8:
|
||||
# e4m3 -> Q dtype is exact; keep the QK dot in Q's dtype (fp8 QK
|
||||
# measured slower here and less accurate).
|
||||
keys = keys.to(query.dtype)
|
||||
if KV_DTYPE == 0:
|
||||
keys = tl.load(
|
||||
k_cache_ptr
|
||||
+ safe_page[None, :] * stride_k_block
|
||||
+ page_offset[None, :] * stride_k_token
|
||||
+ kv_head * stride_k_head
|
||||
+ dim_offsets[:, None],
|
||||
mask=valid[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
values = tl.load(
|
||||
v_cache_ptr
|
||||
+ safe_page[:, None] * stride_v_block
|
||||
+ page_offset[:, None] * stride_v_token
|
||||
+ kv_head * stride_v_head
|
||||
+ dim_offsets[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
elif KV_DTYPE == 1:
|
||||
key_bytes = tl.load(
|
||||
k_cache_ptr
|
||||
+ safe_page[None, :] * stride_k_block
|
||||
+ page_offset[None, :] * stride_k_token
|
||||
+ kv_head * stride_k_head
|
||||
+ dim_offsets[:, None],
|
||||
mask=valid[None, :],
|
||||
other=0,
|
||||
)
|
||||
# UNSCALED: k_scale is folded into the score scaling below.
|
||||
keys = _decode_e4m3_to_bf16(key_bytes)
|
||||
else:
|
||||
keys = tl.load(
|
||||
k_cache_ptr
|
||||
+ safe_page[None, :] * stride_k_block
|
||||
+ page_offset[None, :] * stride_k_token
|
||||
+ kv_head * stride_k_head
|
||||
+ dim_offsets[:, None],
|
||||
mask=valid[None, :],
|
||||
other=0.0,
|
||||
).to(tl.bfloat16)
|
||||
scores = tl.dot(query, keys)
|
||||
# Scaling scores avoids re-quantizing a scaled query to BF16; for fp8
|
||||
# caches the K dequant scale is already folded into softmax_scale on the
|
||||
# host.
|
||||
scores *= score_scale
|
||||
# Scaling scores avoids re-quantizing a scaled query to BF16.
|
||||
if KV_DTYPE == 0:
|
||||
scores *= softmax_scale_log2
|
||||
else:
|
||||
# k_scale is a scalar, so scaling each SCORE is equivalent to
|
||||
# scaling every key element -- but costs one multiply per score
|
||||
# instead of one per element, and skips the fp32 round trip and
|
||||
# the BF16 re-quantization of the scaled key (which for a
|
||||
# non-power-of-two scale also lost mantissa bits). Strictly fewer
|
||||
# roundings than the per-element form it replaces.
|
||||
scores *= softmax_scale_log2 * k_scale_value
|
||||
scores = tl.where(valid[None, :], scores, -1.0e20)
|
||||
next_max = tl.maximum(max_value, tl.max(scores, axis=1))
|
||||
alpha = tl.math.exp2(max_value - next_max)
|
||||
probabilities = tl.where(
|
||||
valid[None, :], tl.math.exp2(scores - next_max[:, None]), 0.0
|
||||
)
|
||||
if IS_FP8:
|
||||
# Dequant V to fp16 (not bf16) for the PV dot: P <= 1 (online
|
||||
# softmax) so fp16 has the range, its wider mantissa is more
|
||||
# accurate, and the fp8->fp16 upcast with an fp16 PV dot is faster.
|
||||
values = values.to(tl.float16)
|
||||
if KV_DTYPE == 1:
|
||||
value_bytes = tl.load(
|
||||
v_cache_ptr
|
||||
+ safe_page[:, None] * stride_v_block
|
||||
+ page_offset[:, None] * stride_v_token
|
||||
+ kv_head * stride_v_head
|
||||
+ dim_offsets[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0,
|
||||
)
|
||||
# UNSCALED: v_scale is folded into the epilogue below.
|
||||
values = _decode_e4m3_to_bf16(value_bytes)
|
||||
elif KV_DTYPE == 2:
|
||||
values = tl.load(
|
||||
v_cache_ptr
|
||||
+ safe_page[:, None] * stride_v_block
|
||||
+ page_offset[:, None] * stride_v_token
|
||||
+ kv_head * stride_v_head
|
||||
+ dim_offsets[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
).to(tl.bfloat16)
|
||||
accumulator = tl.dot(
|
||||
probabilities.to(values.dtype),
|
||||
values,
|
||||
@@ -189,6 +256,13 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||
accumulator * inv_normalizer[:, None],
|
||||
0.0,
|
||||
)
|
||||
if KV_DTYPE != 0:
|
||||
# v_scale folded out of the per-element V decode: the attention output
|
||||
# is linear in V, so one multiply per output element replaces one per
|
||||
# cache element per tile. Safe under split-K too -- every partial is
|
||||
# scaled by the same constant before the LSE merge, which is itself
|
||||
# linear in the partial outputs.
|
||||
normalized_output *= v_scale_value
|
||||
output_mask = head_offsets[:, None] < GROUP_SIZE
|
||||
if NUM_SPLITS == 1:
|
||||
# Preserve the unfused path's BF16 attention-output rounding before
|
||||
@@ -600,17 +674,18 @@ def qsa_sparse_paged_attention(
|
||||
token_to_req: torch.Tensor,
|
||||
use_prefill_config: bool,
|
||||
out: torch.Tensor | None = None,
|
||||
k_scale: float | None = None,
|
||||
v_scale: float | None = None,
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
_force_num_warps: int | None = None,
|
||||
*,
|
||||
output_gate: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Run sparse GQA directly over paged BF16 or FP8-e4m3 K/V caches.
|
||||
"""Run sparse GQA over paged BF16 or FP8 (E4M3 via static scalar scales) K/V caches.
|
||||
|
||||
With fp8 caches, k_scale/v_scale are the layer's per-tensor dequant scales
|
||||
as host floats (e.g. layer._k_scale_float/_v_scale_float): k_scale is
|
||||
pre-multiplied into the softmax scale and v_scale becomes the kernel's
|
||||
output scale, so the kernel needs no device scale buffers.
|
||||
``_force_num_warps`` overrides the tuned launch profile for the warp-count
|
||||
equivalence test only; production callers leave it None. Hardware policy
|
||||
(E4M3 only on CUDA sm_86) is enforced once at Qwen4ExpQSAAttention /
|
||||
Qwen4ExpQSAFlashAttentionImpl construction, not per call.
|
||||
|
||||
logical_indices is the PACKED selection buffer: [rows, selection_width + 1]
|
||||
with the trailing column holding each row's valid-entry count (written by
|
||||
@@ -634,19 +709,26 @@ def qsa_sparse_paged_attention(
|
||||
raise ValueError("QSA sparse attention requires valid grouped-query heads")
|
||||
head_dim = q.shape[2]
|
||||
assert head_dim >= 16 and (head_dim & (head_dim - 1)) == 0
|
||||
assert q.dtype == torch.bfloat16
|
||||
assert k_cache.dtype == v_cache.dtype
|
||||
is_fp8 = k_cache.dtype == torch.float8_e4m3fn
|
||||
if is_fp8:
|
||||
assert k_scale is not None and v_scale is not None
|
||||
# Host pre-multiply: fold the K dequant scale into the attention scale
|
||||
# and pass V's dequant scale as the kernel's output scale.
|
||||
softmax_scale = (head_dim**-0.5) * float(k_scale)
|
||||
output_scale = float(v_scale)
|
||||
if q.dtype != torch.bfloat16:
|
||||
raise ValueError("QSA sparse attention requires BF16 queries")
|
||||
if k_cache.dtype != v_cache.dtype:
|
||||
raise ValueError("QSA sparse attention requires matching K/V dtypes")
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
if k_cache.dtype == torch.bfloat16:
|
||||
kv_dtype = 0
|
||||
elif k_cache.dtype in (torch.uint8, fp8_dtype):
|
||||
kv_dtype = 1
|
||||
# Present platform E4M3 storage as bytes so Triton never materializes
|
||||
# the unsupported fp8e4nv type on SM86.
|
||||
if k_cache.dtype == fp8_dtype:
|
||||
k_cache = k_cache.view(torch.uint8)
|
||||
v_cache = v_cache.view(torch.uint8)
|
||||
elif k_cache.dtype == torch.float8_e5m2:
|
||||
kv_dtype = 2
|
||||
else:
|
||||
assert k_cache.dtype == torch.bfloat16
|
||||
softmax_scale = head_dim**-0.5
|
||||
output_scale = 1.0
|
||||
raise ValueError(
|
||||
"QSA supports only BF16, uint8/E4M3, and E5M2 K/V caches"
|
||||
)
|
||||
assert logical_indices.dtype == block_table.dtype == torch.int32
|
||||
assert token_to_req.dtype == torch.int32
|
||||
assert q.device == k_cache.device == v_cache.device
|
||||
@@ -667,12 +749,37 @@ def qsa_sparse_paged_attention(
|
||||
if not q.shape[0]:
|
||||
return out
|
||||
|
||||
if kv_dtype == 0:
|
||||
# Scale pointers are compile-time dead in the unchanged BF16 kernel.
|
||||
k_scale_ptr = q
|
||||
v_scale_ptr = q
|
||||
else:
|
||||
if k_scale is None:
|
||||
k_scale = torch.tensor(1.0, dtype=torch.float32, device=q.device)
|
||||
if v_scale is None:
|
||||
v_scale = torch.tensor(1.0, dtype=torch.float32, device=q.device)
|
||||
if (
|
||||
k_scale.numel() != 1
|
||||
or v_scale.numel() != 1
|
||||
or k_scale.dtype != torch.float32
|
||||
or v_scale.dtype != torch.float32
|
||||
or k_scale.device != q.device
|
||||
or v_scale.device != q.device
|
||||
):
|
||||
raise ValueError(
|
||||
"QSA K/V scales must be scalar float32 tensors on the query device"
|
||||
)
|
||||
k_scale_ptr = k_scale
|
||||
v_scale_ptr = v_scale
|
||||
|
||||
group_size = q.shape[1] // k_cache.shape[2]
|
||||
block_m = triton.next_power_of_2(group_size)
|
||||
selection_width = logical_indices.shape[1] - 1 # trailing column is the count
|
||||
block_n, partial_warps, num_tiles, num_splits = _select_config(
|
||||
q.shape[0], k_cache.shape[2], use_prefill_config, selection_width, is_fp8
|
||||
)
|
||||
if _force_num_warps is not None:
|
||||
partial_warps = _force_num_warps
|
||||
|
||||
# Split=1 writes output directly and compiles out all workspace accesses.
|
||||
if num_splits == 1:
|
||||
@@ -695,6 +802,8 @@ def qsa_sparse_paged_attention(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
k_scale_ptr,
|
||||
v_scale_ptr,
|
||||
logical_indices,
|
||||
block_table,
|
||||
token_to_req,
|
||||
@@ -729,6 +838,7 @@ def qsa_sparse_paged_attention(
|
||||
NUM_QUERY_HEADS=q.shape[1],
|
||||
NUM_SPLITS=num_splits,
|
||||
NUM_TILES=num_tiles,
|
||||
KV_DTYPE=kv_dtype,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_N=block_n,
|
||||
IS_FP8=is_fp8,
|
||||
@@ -792,13 +902,32 @@ def warmup_qsa_sparse_paged_attention(
|
||||
q_ptr = TritonWarmupTensor(
|
||||
torch.bfloat16, shape=(num_rows, num_query_heads, head_dim)
|
||||
)
|
||||
# Mirror qsa_sparse_paged_attention's dtype dispatch: E4M3 storage is launched
|
||||
# as a uint8 view with scalar float32 scale pointers.
|
||||
if key_cache.dtype == torch.bfloat16:
|
||||
warm_kv_dtype = 0
|
||||
warm_cache_dtype = key_cache.dtype
|
||||
elif key_cache.dtype in (torch.uint8, current_platform.fp8_dtype()):
|
||||
warm_kv_dtype = 1
|
||||
warm_cache_dtype = torch.uint8
|
||||
elif key_cache.dtype == torch.float8_e5m2:
|
||||
warm_kv_dtype = 2
|
||||
warm_cache_dtype = key_cache.dtype
|
||||
else:
|
||||
raise ValueError("QSA warmup: unsupported K/V cache dtype")
|
||||
if warm_kv_dtype == 0:
|
||||
k_scale_ptr = q_ptr
|
||||
v_scale_ptr = q_ptr
|
||||
else:
|
||||
k_scale_ptr = TritonWarmupTensor(torch.float32)
|
||||
v_scale_ptr = TritonWarmupTensor(torch.float32)
|
||||
k_cache_ptr = TritonWarmupTensor(
|
||||
cache_dtype,
|
||||
warm_cache_dtype,
|
||||
shape=tuple(key_cache.shape),
|
||||
strides=tuple(key_cache.stride()),
|
||||
)
|
||||
v_cache_ptr = TritonWarmupTensor(
|
||||
cache_dtype,
|
||||
warm_cache_dtype,
|
||||
shape=tuple(value_cache.shape),
|
||||
strides=tuple(value_cache.stride()),
|
||||
)
|
||||
@@ -838,6 +967,8 @@ def warmup_qsa_sparse_paged_attention(
|
||||
q_ptr,
|
||||
k_cache_ptr,
|
||||
v_cache_ptr,
|
||||
k_scale_ptr,
|
||||
v_scale_ptr,
|
||||
indices_ptr,
|
||||
block_table_ptr,
|
||||
token_to_req_ptr,
|
||||
@@ -872,6 +1003,7 @@ def warmup_qsa_sparse_paged_attention(
|
||||
NUM_QUERY_HEADS=num_query_heads,
|
||||
NUM_SPLITS=num_splits,
|
||||
NUM_TILES=num_tiles,
|
||||
KV_DTYPE=warm_kv_dtype,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_N=block_n,
|
||||
IS_FP8=is_fp8,
|
||||
|
||||
@@ -10,11 +10,126 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker import guard_telemetry
|
||||
|
||||
# Bias so a negative row/index survives atomic_max on a zeroed counter
|
||||
# -- it MUST be a tl.constexpr: a plain module-level int is not readable
|
||||
# from a @triton.jit body -- an earlier build shipped one and every guarded kernel died
|
||||
# with "NameError: Cannot access global variable _GUARD_BIAS" on the smoke.
|
||||
_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS)
|
||||
|
||||
|
||||
_TOPK_WORKSPACE_BYTES = 1024 * 1024
|
||||
_DECODE_BLOCK_N = 64
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MERLIN_QSA_SORT: canonicalise the ORDER of the selected block indices.
|
||||
#
|
||||
# persistent_topk assigns output slots by atomicAdd, so two identical requests
|
||||
# get the same selected SET in a different ORDER (measured on the reference host: matching
|
||||
# sorted hashes, differing raw hashes). Sparse attention then accumulates over
|
||||
# those blocks in whatever order it was handed, and fp32 addition is not
|
||||
# associative, so the arithmetic moves. Sorting is semantically a no-op -- the
|
||||
# same blocks are attended either way -- and makes the accumulation order
|
||||
# canonical.
|
||||
#
|
||||
# Padding (negative sentinels) is pushed to the END so the valid prefix is
|
||||
# preserved: `_expand_qsa_indices_kernel` truncates by POSITION
|
||||
# (`complete_blocks = min(visible_blocks, BLOCK_TOPK)`), and position is never
|
||||
# compared against a score anywhere in the chain, so there is no relevance-rank
|
||||
# contract for the sort to break.
|
||||
#
|
||||
# Branch-free and sync-free on purpose. `.any()` / `.item()` raise
|
||||
# cudaErrorStreamCaptureUnsupported inside graph capture; that cost one boot
|
||||
# on the reference host and is why this version has no Python-visible branch on data.
|
||||
#
|
||||
# Resolved once at import so it is a launch constant: a cudagraph captures the
|
||||
# op sequence that was live at capture time. Code default OFF; vllm/env_override.py
|
||||
# sets 1 (the shipped value) when unset.
|
||||
# ---------------------------------------------------------------------------
|
||||
_QSA_SORT = envs.MERLIN_QSA_SORT
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MERLIN_TOPK_SORTED_EMIT: persistent_topk emits
|
||||
# every row in the canonical order below by itself (sorted_emit_row() in
|
||||
# csrc/libtorch_stable/persistent_topk.cuh), so the Python sort is skipped.
|
||||
#
|
||||
# The decision is NOT a Python parse of the variable. The C++ getter
|
||||
# (topk.cu sorted_emit_enabled) is the single source of truth and publishes
|
||||
# what it cached into persistent_topk_toggle_state() field 6; that is read
|
||||
# once, host-side, right after this process's first persistent_topk launch
|
||||
# (the static is filled by then) and latched. So the sort is dropped only when
|
||||
# the kernel that actually ran says it sorted; an older build (layout 1, no
|
||||
# field 6) or an absent op keeps the Python sort. cooperative_topk (SM90+) has
|
||||
# no sorted emission and always keeps the Python sort.
|
||||
#
|
||||
# MERLIN_TOPK_SORTED_EMIT_CHECK=1 (debug, default off): instead of trusting
|
||||
# the kernel, assert on device that the row already equals its canonical form.
|
||||
# torch._assert_async is capture-safe (no host sync); a violation is a device
|
||||
# assert, i.e. a dead context -- which is the point of a debug check.
|
||||
# ---------------------------------------------------------------------------
|
||||
_SORTED_EMIT_CHECK = envs.MERLIN_TOPK_SORTED_EMIT_CHECK
|
||||
_kernel_sorted_emit_state: bool | None = None # None: not read yet
|
||||
|
||||
|
||||
def _canonical(block_indices: torch.Tensor) -> torch.Tensor:
|
||||
"""Ascending, padding last as -1 -- THE definition of the canonical order,
|
||||
shared by the Python sort and the debug check (and the reference the
|
||||
kernel's sorted emission is tested against)."""
|
||||
big = torch.iinfo(block_indices.dtype).max
|
||||
tmp = torch.where(
|
||||
block_indices < 0, torch.full_like(block_indices, big), block_indices
|
||||
)
|
||||
tmp = torch.sort(tmp, dim=-1).values
|
||||
return torch.where(tmp == big, torch.full_like(tmp, -1), tmp)
|
||||
|
||||
|
||||
def _qsa_canonicalise(block_indices: torch.Tensor) -> None:
|
||||
"""Sort selected block indices ascending, padding last. In place, same shape."""
|
||||
if not _QSA_SORT:
|
||||
return
|
||||
block_indices.copy_(_canonical(block_indices))
|
||||
|
||||
|
||||
def kernel_sorted_emit() -> bool:
|
||||
"""True iff this process's persistent_topk emits canonical order.
|
||||
|
||||
Host-only (a tensor-free query op; no CUDA call), so it is safe inside a
|
||||
capture. Latches once the C++ static is known; an uninitialised static
|
||||
(no launch yet) answers False without latching.
|
||||
"""
|
||||
global _kernel_sorted_emit_state
|
||||
if _kernel_sorted_emit_state is not None:
|
||||
return _kernel_sorted_emit_state
|
||||
op = getattr(torch.ops._C, "persistent_topk_toggle_state", None)
|
||||
if op is None:
|
||||
_kernel_sorted_emit_state = False
|
||||
return False
|
||||
from vllm.v1.worker.e2_toggles import decode_topk_state
|
||||
|
||||
decoded = decode_topk_state(int(op()))
|
||||
if isinstance(decoded, str) or "sorted_emit" not in decoded:
|
||||
# Unknown layout, or layout 1 (no field 6): this build cannot sort.
|
||||
_kernel_sorted_emit_state = False
|
||||
return False
|
||||
value = decoded["sorted_emit"]
|
||||
if value == "uninit":
|
||||
return False # not latched: ask again after a launch
|
||||
_kernel_sorted_emit_state = value == "1"
|
||||
return _kernel_sorted_emit_state
|
||||
|
||||
|
||||
def _check_sorted_emission(block_indices: torch.Tensor) -> None:
|
||||
"""Debug only: device-side assert that the kernel's row is canonical."""
|
||||
if not _SORTED_EMIT_CHECK:
|
||||
return
|
||||
torch._assert_async(
|
||||
torch.eq(block_indices, _canonical(block_indices)).all(),
|
||||
"MERLIN_TOPK_SORTED_EMIT: persistent_topk row is not in canonical order",
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _qsa_mqa_paged_uniform_kernel(
|
||||
q_ptr,
|
||||
@@ -28,6 +143,8 @@ def _qsa_mqa_paged_uniform_kernel(
|
||||
stride_cache_token,
|
||||
stride_table_req,
|
||||
stride_logits_row,
|
||||
num_cache_blocks, # bounds-guard instrumentation
|
||||
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
PAGE_TABLE_WIDTH: tl.constexpr,
|
||||
NUM_HEADS: tl.constexpr,
|
||||
@@ -72,6 +189,11 @@ def _qsa_mqa_paged_uniform_kernel(
|
||||
other=0.0,
|
||||
)
|
||||
column_offsets = tl.arange(0, BLOCK_N)
|
||||
# Bounds-guard instrumentation: accumulate in registers inside the pipelined loop
|
||||
# and do the cross-lane reduction + atomics once after it, so an
|
||||
# atomic-bearing conditional does not sit in the loop body.
|
||||
guard_bad_n = tl.zeros([BLOCK_N], dtype=tl.int32)
|
||||
guard_bad_col = tl.full([BLOCK_N], -1, tl.int32)
|
||||
for tile in tl.range(tile_start, tile_end, num_stages=STAGES):
|
||||
columns = tile * BLOCK_N + column_offsets
|
||||
live = columns < max_visible
|
||||
@@ -80,11 +202,22 @@ def _qsa_mqa_paged_uniform_kernel(
|
||||
physical_page = tl.load(
|
||||
page_table_ptr + request * stride_table_req + logical_page,
|
||||
mask=live,
|
||||
other=0,
|
||||
other=-1,
|
||||
)
|
||||
# Mirror of the QSA attention kernel (qsa.py:137): the VALUE read out of
|
||||
# the page table is untrusted and must be bounded before it becomes an
|
||||
# address. Folds into the existing live mask.
|
||||
page_bad = live & ((physical_page < 0) | (physical_page >= num_cache_blocks))
|
||||
guard_bad_n += page_bad.to(tl.int32)
|
||||
guard_bad_col = tl.maximum(
|
||||
guard_bad_col, tl.where(page_bad, logical_page, -1).to(tl.int32)
|
||||
)
|
||||
live = live & (physical_page >= 0) & (physical_page < num_cache_blocks)
|
||||
# physical_page * block stride can overflow int32 for large caches.
|
||||
safe_page = tl.maximum(physical_page, 0).to(tl.int64)
|
||||
keys = tl.load(
|
||||
k_cache_ptr
|
||||
+ physical_page[:, None].to(tl.int64) * stride_cache_block
|
||||
+ safe_page[:, None] * stride_cache_block
|
||||
+ page_offset[:, None] * stride_cache_token
|
||||
+ dims[None, :],
|
||||
mask=live[:, None] & (dims[None, :] < HEAD_DIM),
|
||||
@@ -106,6 +239,15 @@ def _qsa_mqa_paged_uniform_kernel(
|
||||
& (columns[:, None] < visible[None, :]),
|
||||
)
|
||||
|
||||
# Bounds-guard instrumentation: one reduction and at most one atomic per program.
|
||||
guard_n = tl.sum(guard_bad_n, axis=0)
|
||||
if guard_n > 0:
|
||||
tl.atomic_add(guard_ctr, guard_n.to(tl.int64))
|
||||
tl.atomic_max(guard_ctr + 1, (request).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(
|
||||
guard_ctr + 2, (tl.max(guard_bad_col, axis=0)).to(tl.int64) + _GUARD_BIAS
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_rows", "query_offset"])
|
||||
def _qsa_mqa_paged_prefill_kernel(
|
||||
@@ -124,6 +266,8 @@ def _qsa_mqa_paged_prefill_kernel(
|
||||
num_rows,
|
||||
query_offset,
|
||||
page_table_width,
|
||||
num_cache_blocks, # bounds-guard instrumentation
|
||||
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
NUM_HEADS: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
@@ -181,6 +325,11 @@ def _qsa_mqa_paged_prefill_kernel(
|
||||
other=0.0,
|
||||
)
|
||||
column_offsets = tl.arange(0, BLOCK_N)
|
||||
# Bounds-guard instrumentation: accumulate in registers inside the pipelined loop
|
||||
# and do the cross-lane reduction + atomics once after it, so an
|
||||
# atomic-bearing conditional does not sit in the loop body.
|
||||
guard_bad_n = tl.zeros([BLOCK_N], dtype=tl.int32)
|
||||
guard_bad_col = tl.full([BLOCK_N], -1, tl.int32)
|
||||
for tile in tl.range(k_tile_start, k_tile_end, num_stages=STAGES):
|
||||
columns = tile * BLOCK_N + column_offsets
|
||||
live = columns < max_visible
|
||||
@@ -189,11 +338,22 @@ def _qsa_mqa_paged_prefill_kernel(
|
||||
physical_page = tl.load(
|
||||
page_table_ptr + request * stride_table_req + logical_page,
|
||||
mask=live,
|
||||
other=0,
|
||||
other=-1,
|
||||
)
|
||||
# Mirror of the QSA attention kernel (qsa.py:137): the VALUE read out of
|
||||
# the page table is untrusted and must be bounded before it becomes an
|
||||
# address. Folds into the existing live mask.
|
||||
page_bad = live & ((physical_page < 0) | (physical_page >= num_cache_blocks))
|
||||
guard_bad_n += page_bad.to(tl.int32)
|
||||
guard_bad_col = tl.maximum(
|
||||
guard_bad_col, tl.where(page_bad, logical_page, -1).to(tl.int32)
|
||||
)
|
||||
live = live & (physical_page >= 0) & (physical_page < num_cache_blocks)
|
||||
# physical_page * block stride can overflow int32 for large caches.
|
||||
safe_page = tl.maximum(physical_page, 0).to(tl.int64)
|
||||
keys = tl.load(
|
||||
k_cache_ptr
|
||||
+ physical_page[:, None].to(tl.int64) * stride_cache_block
|
||||
+ safe_page[:, None] * stride_cache_block
|
||||
+ page_offset[:, None] * stride_cache_token
|
||||
+ dims[None, :],
|
||||
mask=live[:, None] & (dims[None, :] < HEAD_DIM),
|
||||
@@ -217,6 +377,15 @@ def _qsa_mqa_paged_prefill_kernel(
|
||||
mask=store_mask,
|
||||
)
|
||||
|
||||
# Bounds-guard instrumentation: one reduction and at most one atomic per program.
|
||||
guard_n = tl.sum(guard_bad_n, axis=0)
|
||||
if guard_n > 0:
|
||||
tl.atomic_add(guard_ctr, guard_n.to(tl.int64))
|
||||
tl.atomic_max(guard_ctr + 1, (request).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(
|
||||
guard_ctr + 2, (tl.max(guard_bad_col, axis=0)).to(tl.int64) + _GUARD_BIAS
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _expand_qsa_indices_kernel(
|
||||
@@ -359,6 +528,8 @@ def warmup_qsa_mqa_paged_decode(
|
||||
k_cache.stride(1),
|
||||
page_table.stride(0),
|
||||
columns,
|
||||
k_cache.shape[0],
|
||||
TritonWarmupTensor(torch.int64),
|
||||
PAGE_SIZE=page_size,
|
||||
PAGE_TABLE_WIDTH=page_table_width,
|
||||
NUM_HEADS=num_heads,
|
||||
@@ -427,6 +598,8 @@ def _prefill_logits(
|
||||
num_queries,
|
||||
query_offset,
|
||||
page_table.shape[1],
|
||||
k_cache.shape[0],
|
||||
guard_telemetry.counter(guard_telemetry.SITE_QSA_PREFILL, q.device),
|
||||
PAGE_SIZE=k_cache.shape[1],
|
||||
NUM_HEADS=q.shape[1],
|
||||
HEAD_DIM=q.shape[2],
|
||||
@@ -502,6 +675,12 @@ def _topk(
|
||||
block_topk,
|
||||
logits.shape[1],
|
||||
)
|
||||
# MERLIN_TOPK_SORTED_EMIT: the kernel already emitted the canonical order,
|
||||
# so the Python sort would be a no-op -- skip it (optionally check it).
|
||||
if not use_cooperative_topk and kernel_sorted_emit():
|
||||
_check_sorted_emission(block_indices)
|
||||
return
|
||||
_qsa_canonicalise(block_indices)
|
||||
|
||||
|
||||
def qsa_select_paged_decode(
|
||||
@@ -554,6 +733,8 @@ def qsa_select_paged_decode(
|
||||
*k_cache.stride()[:2],
|
||||
*page_table.stride()[:-1],
|
||||
*logits.stride()[:-1],
|
||||
k_cache.shape[0],
|
||||
guard_telemetry.counter(guard_telemetry.SITE_QSA_DECODE, q.device),
|
||||
PAGE_SIZE=k_cache.shape[1],
|
||||
PAGE_TABLE_WIDTH=page_table.shape[1],
|
||||
NUM_HEADS=q.shape[1],
|
||||
|
||||
@@ -4,7 +4,27 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.models.qwen4_exp.common.qsa_cache import (
|
||||
QSA_RING_TAG_WORD,
|
||||
QSA_RING_TAIL_ELEMENTS,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker import guard_telemetry
|
||||
|
||||
# Bias so a negative row/index survives atomic_max on a zeroed counter
|
||||
# -- it MUST be a tl.constexpr: a plain module-level int is not readable
|
||||
# from a @triton.jit body -- an earlier build shipped one and every guarded kernel died
|
||||
# with "NameError: Cannot access global variable _GUARD_BIAS" on the smoke.
|
||||
_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS)
|
||||
# Int64 word of the ring tail that carries `logical_position + 1`.
|
||||
_RING_TAG_WORD = tl.constexpr(QSA_RING_TAG_WORD)
|
||||
# Ring-row tag record packing and the encodable range of the found-tag slot.
|
||||
_REC_POS_BITS = tl.constexpr(guard_telemetry.RING_RECORD_POS_BITS)
|
||||
_REC_REQ_BITS = tl.constexpr(guard_telemetry.RING_RECORD_REQ_BITS)
|
||||
_REC_LAYER_BITS = tl.constexpr(guard_telemetry.RING_RECORD_LAYER_BITS)
|
||||
_GUARD_MAX_ENC = tl.constexpr(guard_telemetry.GUARD_MAX_ENCODABLE)
|
||||
_GUARD_MIN_ENC = tl.constexpr(guard_telemetry.GUARD_MIN_ENCODABLE)
|
||||
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -15,6 +35,8 @@ def _norm_rope(
|
||||
pos_w,
|
||||
cos_sin_ptr,
|
||||
cos_sin_stride,
|
||||
cos_sin_rows, # bounds-guard instrumentation
|
||||
guard_ctr, # bounds-guard persistent int64[3], SITE_QSA_ROPE_POS
|
||||
norm_weight_ptr,
|
||||
eps,
|
||||
IS_MROPE: tl.constexpr,
|
||||
@@ -22,6 +44,72 @@ def _norm_rope(
|
||||
MROPE_W: tl.constexpr,
|
||||
):
|
||||
"""Apply Gemma RMSNorm and selected-axis NeoX RoPE to register rows."""
|
||||
# Bounds-guard instrumentation (ring-residue crash replay): the position is a
|
||||
# ROW INDEX into the cos/sin table and nothing bounds it. The masks below
|
||||
# bound which rotary PAIR is read, never which row, so
|
||||
# `cos_sin_ptr + pos * cos_sin_stride` is an address formed from an
|
||||
# unchecked value.
|
||||
#
|
||||
# On the pooled cross-chunk path that value is read straight out of a
|
||||
# raw-key RING row (:305-317). The ring is excluded from zeroing
|
||||
# (single_type_kv_cache_manager.py:96-101), so a re-claimed block keeps the
|
||||
# previous occupant's bytes -- and nothing ties the row the reader selects
|
||||
# (`first_position % STATE_SIZE`) to a row any occupant ever committed, so
|
||||
# it can be allocator residue rather than a position at all. A crash replay drove
|
||||
# exactly this and compute-sanitizer reported 11 invalid reads at
|
||||
# +4.6e18 bytes, the filler bytes decoded as an int64.
|
||||
#
|
||||
# This is the ONLY guard in the series that covers that chain: the request guard
|
||||
# bounds the pre-indexer's `request`, which sits upstream of the ring read
|
||||
# and cannot see a bad position.
|
||||
# WHICH AXIS. Under the production mrope section (11, 11, 10)
|
||||
# all three components are live row indices: the IS_MROPE branch masks
|
||||
# COLUMNS per axis while the row comes unmasked from pos_rows[axis], so a
|
||||
# bad pos_h or pos_w faults exactly as a bad pos_t does. Recording pos_t
|
||||
# unconditionally would name the wrong axis and, for a t-clean/h-bad case,
|
||||
# record an in-range value next to a non-zero count.
|
||||
#
|
||||
# The filler under tl.where must be strongly negative, NOT 0: an offender
|
||||
# that is entirely negative (i_t == -1 is the shape that started this
|
||||
# series) would otherwise max to 0 and read back as position 0 -- the
|
||||
# defect re-entering at the one site with a reproduced fault.
|
||||
_FILL: tl.constexpr = -(1 << 62)
|
||||
bad_t = (pos_t < 0) | (pos_t >= cos_sin_rows)
|
||||
if IS_MROPE:
|
||||
bad_h = (pos_h < 0) | (pos_h >= cos_sin_rows)
|
||||
bad_w = (pos_w < 0) | (pos_w >= cos_sin_rows)
|
||||
else:
|
||||
# the non-mrope branch reads only pos_t (:63-66), and with NO mask at
|
||||
# all, which is why this guard clamps rather than masks.
|
||||
# always-false, but correctly shaped and typed for the max below.
|
||||
bad_h = pos_h < pos_h
|
||||
bad_w = pos_w < pos_w
|
||||
guard_bad = bad_t | bad_h | bad_w
|
||||
guard_n = tl.sum(guard_bad.to(tl.int32), axis=0)
|
||||
if guard_n > 0:
|
||||
worst_t = tl.max(tl.where(bad_t, pos_t, _FILL), axis=0).to(tl.int64)
|
||||
worst_h = tl.max(tl.where(bad_h, pos_h, _FILL), axis=0).to(tl.int64)
|
||||
worst_w = tl.max(tl.where(bad_w, pos_w, _FILL), axis=0).to(tl.int64)
|
||||
worst = tl.maximum(worst_t, tl.maximum(worst_h, worst_w))
|
||||
# slot 2 names the axis: 0 = t, 1 = h, 2 = w. Derived from the MASKS
|
||||
# (deliberately), not by comparing the winner's VALUE against each
|
||||
# axis's max. Value comparison mis-attributes whenever two axes share
|
||||
# the worst value -- which is the common case here, because pos_h and
|
||||
# pos_w are frequently equal to pos_t (the non-2D path sets them so,
|
||||
# and a stale ring tail writes the same int64 into all three). Asking
|
||||
# "did this axis have any offender at all", in t/h/w order, cannot.
|
||||
any_t = tl.max(bad_t.to(tl.int32), axis=0) > 0
|
||||
any_h = tl.max(bad_h.to(tl.int32), axis=0) > 0
|
||||
axis = tl.where(any_t, 0, tl.where(any_h, 1, 2))
|
||||
tl.atomic_add(guard_ctr, guard_n.to(tl.int64))
|
||||
tl.atomic_max(guard_ctr + 1, worst + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, axis.to(tl.int64) + _GUARD_BIAS)
|
||||
# Clamp rather than skip: the rotation is then wrong for these rows, which
|
||||
# is what a masked fault IS. The counter is the result, not the output.
|
||||
pos_t = tl.minimum(tl.maximum(pos_t, 0), cos_sin_rows - 1)
|
||||
pos_h = tl.minimum(tl.maximum(pos_h, 0), cos_sin_rows - 1)
|
||||
pos_w = tl.minimum(tl.maximum(pos_w, 0), cos_sin_rows - 1)
|
||||
|
||||
TILE_T: tl.constexpr = x.shape[0]
|
||||
TILE_H: tl.constexpr = x.shape[1]
|
||||
D: tl.constexpr = x.shape[2]
|
||||
@@ -107,6 +195,8 @@ def _store_rotated(dst, y, o1, o2, scale):
|
||||
"num_state_blocks",
|
||||
"num_compressed_blocks",
|
||||
"num_k_work",
|
||||
"num_requests",
|
||||
"layer_id",
|
||||
]
|
||||
)
|
||||
def _qsa_prepare_kernel(
|
||||
@@ -118,6 +208,8 @@ def _qsa_prepare_kernel(
|
||||
pos_stride_axis,
|
||||
pos_stride_token,
|
||||
cos_sin_ptr,
|
||||
cos_sin_rows, # bounds-guard instrumentation
|
||||
rope_guard_ctr, # bounds-guard persistent int64[3], SITE_QSA_ROPE_POS
|
||||
q_norm_weight_ptr,
|
||||
k_norm_weight_ptr,
|
||||
eps,
|
||||
@@ -141,6 +233,11 @@ def _qsa_prepare_kernel(
|
||||
num_state_blocks,
|
||||
num_compressed_blocks,
|
||||
num_k_work,
|
||||
num_requests, # bounds-guard instrumentation
|
||||
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||
ring_skip_ctr, # ring-skip telemetry, persistent int64[3], SITE_QSA_RING_SKIP
|
||||
ring_uncommitted_ctr, # ring-row tag fix, persistent int64[3], SITE_QSA_RING_UNCOMMITTED
|
||||
layer_id, # ring-row tag fix: named in the uncommitted-row record only
|
||||
HQ: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
TILE_T_Q: tl.constexpr,
|
||||
@@ -278,6 +375,8 @@ def _qsa_prepare_kernel(
|
||||
pos_w,
|
||||
cos_sin_ptr,
|
||||
D // 2,
|
||||
cos_sin_rows,
|
||||
rope_guard_ctr,
|
||||
q_norm_weight_ptr,
|
||||
eps,
|
||||
IS_2D_POSITIONS,
|
||||
@@ -300,6 +399,17 @@ def _qsa_prepare_kernel(
|
||||
request, work_in_request = tl.split(work_metadata)
|
||||
if request < 0:
|
||||
return
|
||||
# Second bounds guard: the early return above rejects only the -1
|
||||
# sentinel the builder writes for inactive work items. Any other
|
||||
# corrupted value was loaded straight out of query_start_loc, which has
|
||||
# num_requests + 1 entries -- 8 bytes for a single-request batch. The
|
||||
# chunk_end load below derives its index from query_end, so a bad row
|
||||
# also walked logical_positions; returning here stops both.
|
||||
if request >= num_requests:
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (request).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, (work_in_request).to(tl.int64) + _GUARD_BIAS)
|
||||
return
|
||||
|
||||
query_start = tl.load(query_start_loc_ptr + request)
|
||||
query_end = tl.load(query_start_loc_ptr + request + 1)
|
||||
@@ -351,6 +461,64 @@ def _qsa_prepare_kernel(
|
||||
+ safe_state_block * state_cache_stride_block
|
||||
+ (source_positions % STATE_SIZE)[:, None] * state_cache_stride_token
|
||||
)
|
||||
# Ring-row tag fix: EVERY historical source row must carry the tag the
|
||||
# writer stores for exactly this logical position, before anything
|
||||
# reads its key or its position tail. A missing ring block (state
|
||||
# table entry out of range) reads tag 0 and fails the same way --
|
||||
# before the ring-row tag fix it pooled zeros silently. In-chunk rows come from
|
||||
# this launch's own k and are never checked.
|
||||
historical = valid & ~source_in_chunk
|
||||
source_tags = tl.load(
|
||||
(
|
||||
state_cache_ptr
|
||||
+ safe_state_block * state_cache_stride_block
|
||||
+ (source_positions % STATE_SIZE) * state_cache_stride_token
|
||||
+ D
|
||||
).to(tl.pointer_type(tl.int64))
|
||||
+ _RING_TAG_WORD,
|
||||
mask=historical & state_block_valid,
|
||||
other=0,
|
||||
)
|
||||
row_bad = historical & (source_tags != source_positions.to(tl.int64) + 1)
|
||||
num_bad_rows = tl.sum(row_bad.to(tl.int32), axis=0)
|
||||
group_ok = num_bad_rows == 0
|
||||
if num_bad_rows > 0:
|
||||
# One coherent record per site, first offender wins: the CAS
|
||||
# claims slot 1 and only its winner writes slot 2, so both
|
||||
# fields always describe the same row.
|
||||
bad_position = tl.min(
|
||||
tl.where(row_bad, source_positions.to(tl.int64), 1 << 62),
|
||||
axis=0,
|
||||
)
|
||||
bad_tag = tl.sum(
|
||||
tl.where(
|
||||
row_bad & (source_positions.to(tl.int64) == bad_position),
|
||||
source_tags,
|
||||
0,
|
||||
),
|
||||
axis=0,
|
||||
)
|
||||
record = (
|
||||
(
|
||||
((layer_id + 1).to(tl.int64) & ((1 << _REC_LAYER_BITS) - 1))
|
||||
<< (_REC_REQ_BITS + _REC_POS_BITS)
|
||||
)
|
||||
| (
|
||||
(request.to(tl.int64) & ((1 << _REC_REQ_BITS) - 1))
|
||||
<< _REC_POS_BITS
|
||||
)
|
||||
| (bad_position & ((1 << _REC_POS_BITS) - 1))
|
||||
)
|
||||
tl.atomic_add(ring_uncommitted_ctr, num_bad_rows.to(tl.int64))
|
||||
zero64 = bad_position * 0
|
||||
prev = tl.atomic_cas(
|
||||
ring_uncommitted_ctr + 1, zero64, record + _GUARD_BIAS
|
||||
)
|
||||
if prev == 0:
|
||||
clamped_tag = tl.minimum(
|
||||
tl.maximum(bad_tag, _GUARD_MIN_ENC), _GUARD_MAX_ENC
|
||||
)
|
||||
tl.atomic_xchg(ring_uncommitted_ctr + 2, clamped_tag + _GUARD_BIAS)
|
||||
# Only the first completed group can cross the chunk boundary. Select
|
||||
# historical rows from the ring without issuing two masked loads.
|
||||
source_base = tl.where(source_in_chunk[:, None], current_base, cached_base)
|
||||
@@ -359,9 +527,11 @@ def _qsa_prepare_kernel(
|
||||
source_valid = tl.where(
|
||||
source_in_chunk, source_tokens_valid, state_block_valid
|
||||
)
|
||||
# A failed group loads nothing (the mask, not a tl.where on the
|
||||
# result, is what suppresses the memory operation).
|
||||
source = tl.load(
|
||||
source_base + dims[None, :],
|
||||
mask=valid & source_valid[:, None],
|
||||
mask=valid & group_ok & source_valid[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
# Match the unfused path's BF16 pooled tensor before RMSNorm.
|
||||
@@ -409,7 +579,9 @@ def _qsa_prepare_kernel(
|
||||
+ (first_position % STATE_SIZE) * state_cache_stride_token
|
||||
+ D
|
||||
).to(tl.pointer_type(tl.int64))
|
||||
load_cached_position = ~first_in_chunk & state_block_valid
|
||||
# The first row's tag was validated above with the rest;
|
||||
# a failed group never reads its tail (pos stays 0, in range).
|
||||
load_cached_position = ~first_in_chunk & state_block_valid & group_ok
|
||||
cached_pos_t = tl.load(tail, mask=load_cached_position, other=0)
|
||||
cached_pos_h = tl.load(tail + 1, mask=load_cached_position, other=0)
|
||||
cached_pos_w = tl.load(tail + 2, mask=load_cached_position, other=0)
|
||||
@@ -427,6 +599,8 @@ def _qsa_prepare_kernel(
|
||||
pos_w + tl.arange(0, 1),
|
||||
cos_sin_ptr,
|
||||
D // 2,
|
||||
cos_sin_rows,
|
||||
rope_guard_ctr,
|
||||
k_norm_weight_ptr,
|
||||
eps,
|
||||
IS_K_MROPE,
|
||||
@@ -435,13 +609,16 @@ def _qsa_prepare_kernel(
|
||||
)
|
||||
compressed_block = (compressed_slot // COMP_PAGE_SIZE).to(tl.int64)
|
||||
compressed_row = compressed_slot % COMP_PAGE_SIZE
|
||||
# A group built from an uncommitted row must never become
|
||||
# usable compressed state. The slot stays unwritten; the site is
|
||||
# fatal because the selection kernels will still read it.
|
||||
tl.store(
|
||||
compressed_cache_ptr
|
||||
+ compressed_block * compressed_cache_stride_block
|
||||
+ compressed_row * compressed_cache_stride_token
|
||||
+ dims,
|
||||
tl.reshape(y, (D,)),
|
||||
mask=valid,
|
||||
mask=valid & group_ok,
|
||||
)
|
||||
|
||||
if work_in_request == 0:
|
||||
@@ -459,6 +636,19 @@ def _qsa_prepare_kernel(
|
||||
valid_slot = (
|
||||
valid_token & (slot >= 0) & (slot < num_state_blocks * STATE_SIZE)
|
||||
)
|
||||
# Ring-skip telemetry: a live token whose slot fails the range check
|
||||
# has its ring row silently NOT written (every store below is
|
||||
# masked by valid_slot). That token's raw key is then absent
|
||||
# from the ring the next step reads (a candidate cause). Count
|
||||
# it -- the write stays skipped, the reconciliation is the ring-row tag fix.
|
||||
skipped = valid_token & ~valid_slot
|
||||
tl.atomic_add(ring_skip_ctr, 1, mask=skipped)
|
||||
tl.atomic_max(
|
||||
ring_skip_ctr + 1, token.to(tl.int64) + _GUARD_BIAS, mask=skipped
|
||||
)
|
||||
tl.atomic_max(
|
||||
ring_skip_ctr + 2, slot.to(tl.int64) + _GUARD_BIAS, mask=skipped
|
||||
)
|
||||
safe_slot = tl.maximum(slot, 0)
|
||||
state_row = (
|
||||
state_cache_ptr
|
||||
@@ -499,6 +689,20 @@ def _qsa_prepare_kernel(
|
||||
tl.store(tail, pos_t.to(tl.int64), mask=valid_slot)
|
||||
tl.store(tail + 1, pos_h.to(tl.int64), mask=valid_slot)
|
||||
tl.store(tail + 2, pos_w.to(tl.int64), mask=valid_slot)
|
||||
# Ring-row tag fix: the tag names the logical position this row now
|
||||
# holds, taken from the authoritative per-token metadata (not
|
||||
# re-derived from chunk arithmetic), stored after key + tail.
|
||||
# Every committed row gets one, with or without RoPE positions.
|
||||
token_logical = tl.load(
|
||||
logical_positions_ptr + tl.maximum(token, 0),
|
||||
mask=valid_slot,
|
||||
other=-1,
|
||||
)
|
||||
tl.store(
|
||||
(state_row + D).to(tl.pointer_type(tl.int64)) + _RING_TAG_WORD,
|
||||
token_logical.to(tl.int64) + 1,
|
||||
mask=valid_slot,
|
||||
)
|
||||
|
||||
|
||||
def qsa_prepare(
|
||||
@@ -530,11 +734,16 @@ def qsa_prepare(
|
||||
main_slot_mapping: torch.Tensor,
|
||||
main_k_scale: float,
|
||||
main_v_scale: float,
|
||||
layer_id: int = -1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Normalize Q, compress K, then update the circular raw state.
|
||||
|
||||
Also prepares the main attention (QK-norm/RoPE, gate copy, K/V cache write)
|
||||
and returns its Q and gate.
|
||||
|
||||
``state_cache`` rows are ``[key D bf16 | 4 int64: pos_t, pos_h, pos_w, tag]``
|
||||
(``QSAKeyStateCache``, ring-row tag fix); the tail is present with or without
|
||||
``rope_pos_offset``. ``layer_id`` only labels the uncommitted-row record.
|
||||
"""
|
||||
num_tokens = q.shape[0]
|
||||
main_head_dim = main_kv_cache.shape[-1] // 2
|
||||
@@ -559,6 +768,15 @@ def qsa_prepare(
|
||||
is_k_mrope = bool(mrope_section)
|
||||
cache_has_rope_pos = rope_pos_offset is not None
|
||||
assert rope_pos_offset is None or rope_pos_offset == head_dim
|
||||
# The kernel addresses the tail at +head_dim and the tag at word 3 of
|
||||
# it. A ring without the four-word tail (the older D or D+12 layouts)
|
||||
# would put the tag store into the NEXT row, so refuse it outright.
|
||||
assert state_cache.shape[-1] == head_dim + QSA_RING_TAIL_ELEMENTS, (
|
||||
f"QSA ring row width {state_cache.shape[-1]} != head_dim + "
|
||||
f"{QSA_RING_TAIL_ELEMENTS} (key + pos_t/h/w + tag)"
|
||||
)
|
||||
assert state_cache.dtype == torch.bfloat16
|
||||
assert state_cache.stride(1) % 4 == 0 and state_cache.stride(0) % 4 == 0
|
||||
if is_2d_positions:
|
||||
assert positions.shape == (3, num_tokens)
|
||||
assert is_k_mrope
|
||||
@@ -589,6 +807,8 @@ def qsa_prepare(
|
||||
pos_stride_axis,
|
||||
pos_stride_token,
|
||||
cos_sin_cache,
|
||||
cos_sin_cache.shape[0],
|
||||
guard_telemetry.counter(guard_telemetry.SITE_QSA_ROPE_POS, q.device),
|
||||
q_norm_weight,
|
||||
k_norm_weight,
|
||||
eps,
|
||||
@@ -612,6 +832,11 @@ def qsa_prepare(
|
||||
state_cache.shape[0],
|
||||
compressed_cache.shape[0],
|
||||
num_k_work,
|
||||
min(query_start_loc.numel() - 1, state_block_table.shape[0]),
|
||||
guard_telemetry.counter(guard_telemetry.SITE_QSA_PRE_INDEXER, q.device),
|
||||
guard_telemetry.counter(guard_telemetry.SITE_QSA_RING_SKIP, q.device),
|
||||
guard_telemetry.counter(guard_telemetry.SITE_QSA_RING_UNCOMMITTED, q.device),
|
||||
int(layer_id),
|
||||
HQ=num_q_heads,
|
||||
D=head_dim,
|
||||
TILE_T_Q=TILE_T_Q,
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Python reader for the medium-path exact-restart top-k counters.
|
||||
|
||||
The counters themselves are device-global uint64s owned by
|
||||
`csrc/libtorch_stable/topk.cu`; `torch.ops._C.persistent_topk_exact_counters`
|
||||
snapshots them into an **int64** CUDA tensor. Without this module nothing in a
|
||||
running engine ever calls that op, so the instrument records into a buffer no
|
||||
one reads -- which measures nothing.
|
||||
|
||||
DTYPE IS PART OF THE PROTOCOL (large-path determinism change). Version 1 was u32
|
||||
device counters read into a signed int32 tensor, which goes negative after
|
||||
~33k all-equal rows at k=512 and wraps after ~66k, so a delta taken across the
|
||||
boundary is a wrong number that looks like a measurement. Version 2 widens
|
||||
both sides, and the op REJECTS an int32 buffer rather than filling it from half
|
||||
of each counter. An older build whose op still demands int32 will therefore
|
||||
raise here; that is the intended signal, not a regression to paper over.
|
||||
|
||||
SLOT LAYOUT is build-dependent, so nothing here hard-codes five. `slot_count()`
|
||||
asks the build first -- the tie-determinism change added the `..._slots` op this was written to look
|
||||
for, so on a build that has it the env variable is never consulted -- and
|
||||
otherwise reads `MERLIN_TOPK_COUNTER_SLOTS` (default 5). The op demands
|
||||
EXACTLY that many elements, so a guessed length is an error, not a short read;
|
||||
names come from a table keyed by that length:
|
||||
|
||||
5 rows overflow fallbacks faults switch
|
||||
10 rows overflow fallbacks faults rows_large ties_cut ties_dropped
|
||||
faults_large switch switch_large
|
||||
11 the ten above, plus decode_overflow
|
||||
|
||||
An unknown length degrades to `slot<i>` names rather than mislabelling.
|
||||
|
||||
WHY 11 AND NOT 10. The tie-determinism change numbered its one new counter around the large-path block
|
||||
instead of after the medium-path one, so the integration renumbers nothing: slots 4-7 are
|
||||
the large-path counters, 8-9 the two switch flags, 10 the decode overflow.
|
||||
On the integrated build every one of the eleven is live; on a single-branch
|
||||
build the other branch's slots are structurally zero and the line still
|
||||
reports them by name -- they are the slots they will hold, not a
|
||||
mislabelling. `decode_overflow` is the number that has to be read before any
|
||||
claim that the DECODE path is deterministic: it counts rows whose buffered
|
||||
candidates exceeded the 3,708-slot capacity before the tie ranking ran, which
|
||||
ranking cannot repair.
|
||||
|
||||
HOW THE SMOKE READS IT. Set `MERLIN_TOPK_EXACT_COUNTERS_LOG=<seconds>` on the
|
||||
engine; each GPU worker then emits, at most that often and once more at
|
||||
shutdown, one line into the ordinary engine log:
|
||||
|
||||
topk-exact-counters rank=PP0_TP1 rows=... overflow=... fallbacks=...
|
||||
faults=... switch=... [rows_large=... ... decode_overflow=...]
|
||||
|
||||
The first five fields are the medium-path line and keep their order on every build;
|
||||
any further slot is appended after them, so an 11-slot build reads as the medium-path
|
||||
line plus the four large-path fields, `switch_large` and `decode_overflow`.
|
||||
|
||||
Counts are cumulative per worker process, so the smoke takes the LATEST line
|
||||
per rank -- `grep topk-exact-counters <engine log> | ...` keyed on the `rank=`
|
||||
field -- and reads `faults` as the pass/fail number (it must stay 0) with
|
||||
`rows` as the denominator. Earlier lines are only useful as a rate.
|
||||
|
||||
COST. `read()` passes `sync=1`, i.e. a full device synchronize, because a
|
||||
stream-ordered snapshot can miss increments from work in flight on another
|
||||
stream. That is acceptable ONLY because the cadence is coarse (>= 30 s in
|
||||
practice): the time check happens before the op call, so a step that is not
|
||||
due for a log touches nothing but `time.monotonic()`. The read is also refused
|
||||
inside a cudagraph capture -- the op allocates and synchronizes, which would
|
||||
abort the capture.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_OP_NAME = "persistent_topk_exact_counters"
|
||||
|
||||
_MEDIUM_NAMES = ("rows", "overflow", "fallbacks", "faults", "switch")
|
||||
_LARGEDET_NAMES = (
|
||||
"rows",
|
||||
"overflow",
|
||||
"fallbacks",
|
||||
"faults",
|
||||
"rows_large",
|
||||
"ties_cut",
|
||||
"ties_dropped",
|
||||
"faults_large",
|
||||
"switch",
|
||||
"switch_large",
|
||||
)
|
||||
# The decode-overflow counter comes after the large-path block; see the module
|
||||
# docstring for why it is slot 10 and not slot 4.
|
||||
_TIEDET_NAMES = _LARGEDET_NAMES + ("decode_overflow",)
|
||||
_SLOT_NAMES: dict[int, tuple[str, ...]] = {
|
||||
5: _MEDIUM_NAMES,
|
||||
10: _LARGEDET_NAMES,
|
||||
11: _TIEDET_NAMES,
|
||||
}
|
||||
|
||||
# The medium-path line contract. Any further name is appended after these, in slot
|
||||
# order, so a 10-slot build still starts with exactly the medium-path line.
|
||||
_LINE_ORDER = _MEDIUM_NAMES
|
||||
|
||||
_warned_absent = False
|
||||
_warned_failed = False
|
||||
_last_log_monotonic = 0.0
|
||||
|
||||
|
||||
def _get_op(name: str):
|
||||
try:
|
||||
return getattr(torch.ops._C, name, None)
|
||||
except (AttributeError, RuntimeError):
|
||||
return None
|
||||
|
||||
|
||||
def slot_count() -> int:
|
||||
"""How many int64 slots this build's counter array carries.
|
||||
|
||||
The build is asked first and the environment is only a fallback for builds
|
||||
older than the slot-count op: a guessed length that does not match the extension is
|
||||
rejected by the op's own length check, so the instrument would go dark
|
||||
exactly when the layout changed -- which is when it matters most.
|
||||
"""
|
||||
query = _get_op(_OP_NAME + "_slots")
|
||||
if query is not None:
|
||||
return int(query())
|
||||
return int(os.getenv("MERLIN_TOPK_COUNTER_SLOTS", "5"))
|
||||
|
||||
|
||||
def slot_names(count: int) -> tuple[str, ...]:
|
||||
names = _SLOT_NAMES.get(count)
|
||||
return names if names is not None else tuple(f"slot{i}" for i in range(count))
|
||||
|
||||
|
||||
def read(reset: bool = False, sync: bool = True) -> dict[str, int]:
|
||||
"""Snapshot the counters as {slot name: value}; {} when unreadable."""
|
||||
global _warned_absent
|
||||
op = _get_op(_OP_NAME)
|
||||
if op is None:
|
||||
if not _warned_absent:
|
||||
_warned_absent = True
|
||||
logger.warning(
|
||||
"torch.ops._C.%s is not in this build; the top-k exact-restart "
|
||||
"counters cannot be read.",
|
||||
_OP_NAME,
|
||||
)
|
||||
return {}
|
||||
if reset and not sync:
|
||||
raise ValueError("reset requires sync: zeroing races in-flight atomics")
|
||||
if not (torch.cuda.is_available() and torch.cuda.is_initialized()):
|
||||
return {}
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
# The op allocates and synchronizes; calling it here aborts the capture.
|
||||
return {}
|
||||
count = slot_count()
|
||||
device = torch.device("cuda", torch.cuda.current_device())
|
||||
# int64: the device counters are 64-bit and the op refuses any other
|
||||
# width. See the module docstring -- this is protocol version 2.
|
||||
out = torch.zeros(count, dtype=torch.int64, device=device)
|
||||
op(out, 1 if reset else 0, 1 if sync else 0)
|
||||
return dict(zip(slot_names(count), out.tolist(), strict=True))
|
||||
|
||||
|
||||
def rank_tag() -> str:
|
||||
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
|
||||
|
||||
try:
|
||||
return f"PP{get_pp_group().rank_in_group}_TP{get_tp_group().rank_in_group}"
|
||||
except Exception:
|
||||
# Pre-init or single-process: say so rather than claim rank 0.
|
||||
return "PP?_TP?"
|
||||
|
||||
|
||||
def log_line(tag: str) -> str:
|
||||
"""The one-line form the smoke greps; "" when the counters are unreadable."""
|
||||
counters = read()
|
||||
if not counters:
|
||||
return ""
|
||||
order = [name for name in _LINE_ORDER if name in counters]
|
||||
order += [name for name in counters if name not in _LINE_ORDER]
|
||||
fields = " ".join(f"{name}={counters[name]}" for name in order)
|
||||
return f"topk-exact-counters rank={tag} {fields}"
|
||||
|
||||
|
||||
def maybe_log(force: bool = False) -> str:
|
||||
"""Emit the line if due (or forced). Time check first: when not due this
|
||||
costs one clock read and never touches the device."""
|
||||
global _last_log_monotonic, _warned_failed
|
||||
period = envs.MERLIN_TOPK_EXACT_COUNTERS_LOG
|
||||
if period <= 0:
|
||||
return ""
|
||||
now = time.monotonic()
|
||||
if not force and now - _last_log_monotonic < period:
|
||||
return ""
|
||||
_last_log_monotonic = now
|
||||
try:
|
||||
line = log_line(rank_tag())
|
||||
except Exception as exc: # never let an instrument break a step or shutdown
|
||||
if not _warned_failed:
|
||||
_warned_failed = True
|
||||
logger.warning("top-k exact-restart counter read failed: %s", exc)
|
||||
return ""
|
||||
if line:
|
||||
logger.info("%s", line)
|
||||
return line
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context
|
||||
@@ -18,6 +19,7 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
PleShortConvAttentionBackend,
|
||||
@@ -91,16 +93,37 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
self.conv_state_len = (self.conv_kernel_size - 1) * self.short_conv_dilation
|
||||
self.num_spec_tokens = vllm_config.num_speculative_tokens
|
||||
self.activation = "silu"
|
||||
self.ple_embedding = Qwen4ExpNGramEmbedding(
|
||||
config,
|
||||
int(config.ple_embed_dim),
|
||||
self.ple_dense_layer_id,
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
|
||||
prefix=f"{prefix}.ple_embedding",
|
||||
quant_config=quant_config,
|
||||
params_dtype=model_config.dtype,
|
||||
)
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
# Fork: the E2 pull transport owns the table in a CPU-only
|
||||
# offload process; upstream's device/UVA backends are never built.
|
||||
# Pull transport with the compile-profile guard: no layer_name -- the pull module never routes
|
||||
# through the n-gram custom ops. Its GPU placeholder dispatches
|
||||
# through PleOffloadLayer.forward (the ple_offload_wait_copy op) and
|
||||
# owns no table; the offload process calls forward_impl directly.
|
||||
from .ple_pull import Qwen4ExpNGramEmbeddingPull
|
||||
|
||||
self.ple_embedding = Qwen4ExpNGramEmbeddingPull.build(
|
||||
config,
|
||||
int(config.ple_embed_dim),
|
||||
self.ple_dense_layer_id,
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
|
||||
prefix=f"{prefix}.ple_embedding",
|
||||
quant_config=quant_config,
|
||||
params_dtype=model_config.dtype,
|
||||
)
|
||||
else:
|
||||
self.ple_embedding = Qwen4ExpNGramEmbedding(
|
||||
config,
|
||||
int(config.ple_embed_dim),
|
||||
self.ple_dense_layer_id,
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
|
||||
prefix=f"{prefix}.ple_embedding",
|
||||
layer_name=prefix,
|
||||
quant_config=quant_config,
|
||||
params_dtype=model_config.dtype,
|
||||
)
|
||||
# The PLE cache is TP-replicated, so this merged projection is too.
|
||||
self.kv_proj = MergedColumnParallelLinear(
|
||||
int(config.ple_embed_dim),
|
||||
@@ -144,6 +167,11 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
output_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize PLE lookup output."""
|
||||
|
||||
dequantize_pulled = getattr(self.ple_embedding, "dequantize_pulled", None)
|
||||
if dequantize_pulled is not None:
|
||||
# Fork pull transport: no local table; FP8 scale kept here.
|
||||
return dequantize_pulled(embeddings, output_dtype)
|
||||
return self.ple_embedding.ngram_embedding.dequantize(
|
||||
embeddings,
|
||||
output_dtype,
|
||||
@@ -429,11 +457,44 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
self.norm_key.eps,
|
||||
)
|
||||
# State routing depends on runtime request metadata and remains outside
|
||||
# the graph; short convolution accumulates into gated_output.
|
||||
self._short_conv(conv_input, gated_output, hidden_states)
|
||||
# the graph; short convolution accumulates into gated_output. Compile-profile guard:
|
||||
# a splitting op again (as on v2.0.1), so VLLM_COMPILE never traces it.
|
||||
torch.ops.vllm.qwen4_exp_ple_short_conv(
|
||||
conv_input, gated_output, hidden_states, self.prefix
|
||||
)
|
||||
return gated_output
|
||||
|
||||
|
||||
def qwen4_exp_ple_short_conv(
|
||||
inputs: torch.Tensor,
|
||||
residual_output: torch.Tensor,
|
||||
outer_residual: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""PLE short convolution + state routing, outside piecewise graphs."""
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
if not isinstance(layer, Qwen4ExpPLELayer):
|
||||
raise TypeError(f"{layer_name} is not a Qwen4Exp PLE layer")
|
||||
layer._short_conv(inputs, residual_output, outer_residual)
|
||||
|
||||
|
||||
def _qwen4_exp_ple_short_conv_fake(
|
||||
inputs: torch.Tensor,
|
||||
residual_output: torch.Tensor,
|
||||
outer_residual: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_ple_short_conv",
|
||||
op_func=qwen4_exp_ple_short_conv,
|
||||
mutates_args=["residual_output"],
|
||||
fake_impl=_qwen4_exp_ple_short_conv_fake,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Qwen4ExpPLEGroupedNorm",
|
||||
"Qwen4ExpPLELayer",
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The fork's PLE pull transport, re-seated on vLLM v0.30.0.
|
||||
|
||||
v0.30.0 split the E2 base's offload-aware ``Qwen4ExpNGramEmbedding`` into an
|
||||
upstream-only module (``ngram_embedding.py``, device or pinned-host UVA table).
|
||||
This module puts the E2 pull transport back WITHOUT editing that upstream
|
||||
class: ``Qwen4ExpNGramEmbeddingPull`` inherits it and ``PleOffloadLayer``.
|
||||
|
||||
* In a GPU worker, ``PleOffloadLayer.__init_subclass__`` skips the constructor:
|
||||
the module is an empty placeholder with no table. ``PleOffloadConnector``
|
||||
binds it to a host-mapped flag/buffer; its forward is the captured
|
||||
``ple_offload_wait_copy`` op. Only the FP8 global scale is kept on the GPU.
|
||||
* In the CPU-only offload process the constructor runs in full, on CPU. The
|
||||
table is upstream's ``Qwen4ExpPLEDeviceEmbedding`` allocated on CPU;
|
||||
``forward_impl`` computes n-gram IDs and gathers raw (FP8) rows.
|
||||
|
||||
Selected when ``VLLM_E2_PLE_PULL_TRANSPORT`` is ``1`` or unset (the fork default
|
||||
since v2.5); with ``=0`` this module is never imported by the model and the
|
||||
tree is upstream.
|
||||
|
||||
Supported table formats: FP8 (global scale) and unquantized -- the two
|
||||
``Qwen4ExpPLEEmbeddingMethod`` families v0.30.0 itself supports. The E2 NVFP4
|
||||
path is not carried (v0.30.0 has no NVFP4 PLE method to select).
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.ple_offload_layer import (
|
||||
PleOffloadLayer,
|
||||
is_offload_process,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from .ngram_embedding import (
|
||||
Qwen4ExpNGramEmbedding,
|
||||
Qwen4ExpPLEEmbeddingMethod,
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
#: The backend name this module reports in the PLE producer log line.
|
||||
PULL_BACKEND_NAME = "E2PullTransport"
|
||||
|
||||
|
||||
class Qwen4ExpNGramEmbeddingPull(PleOffloadLayer, Qwen4ExpNGramEmbedding):
|
||||
"""Upstream n-gram embedding whose table lives in the offload process."""
|
||||
|
||||
#: GPU placeholder: the method the offload process will use for the table,
|
||||
#: resolved with upstream's own selector (set by ``build``).
|
||||
_offload_quant_method: Qwen4ExpPLEEmbeddingMethod | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen4ExpTextConfig,
|
||||
embedding_dim: int,
|
||||
ple_dense_layer_id: int,
|
||||
max_total_tokens: int,
|
||||
*,
|
||||
data_parallel_rank: int,
|
||||
prefix: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
) -> None:
|
||||
# Runs only in the offload process (guarded by PleOffloadLayer).
|
||||
engram_config = get_current_vllm_config().engram_config
|
||||
if engram_config is not None and engram_config.cpu_offload:
|
||||
raise RuntimeError(
|
||||
"PLE pull transport: the offload process resolved "
|
||||
"cpu_offload=True; refusing to build upstream's UVA table here."
|
||||
)
|
||||
Qwen4ExpNGramEmbedding.__init__(
|
||||
self,
|
||||
config,
|
||||
embedding_dim,
|
||||
ple_dense_layer_id,
|
||||
max_total_tokens,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
prefix=prefix,
|
||||
quant_config=quant_config,
|
||||
params_dtype=params_dtype,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build(
|
||||
cls,
|
||||
config: Qwen4ExpTextConfig,
|
||||
embedding_dim: int,
|
||||
ple_dense_layer_id: int,
|
||||
max_total_tokens: int,
|
||||
*,
|
||||
data_parallel_rank: int,
|
||||
prefix: str,
|
||||
quant_config: QuantizationConfig | None,
|
||||
params_dtype: torch.dtype | None,
|
||||
) -> "Qwen4ExpNGramEmbeddingPull":
|
||||
"""Construct the placeholder (GPU) or the full CPU module (offload)."""
|
||||
# The offload process builds the surrounding model on meta; this
|
||||
# subtree must own real CPU storage. In a GPU worker the constructor is
|
||||
# skipped, so the device context allocates nothing.
|
||||
with torch.device(cls.get_target_device()):
|
||||
module = cls(
|
||||
config,
|
||||
embedding_dim,
|
||||
ple_dense_layer_id,
|
||||
max_total_tokens,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
prefix=prefix,
|
||||
quant_config=quant_config,
|
||||
params_dtype=params_dtype,
|
||||
)
|
||||
embedding_prefix = f"{prefix}.ngram_embedding"
|
||||
method = Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
embedding_prefix,
|
||||
getattr(config, "ple_embedding_dtype", None),
|
||||
)
|
||||
if not isinstance(
|
||||
method,
|
||||
(Qwen4ExpPLEFp8EmbeddingMethod, Qwen4ExpPLEUnquantizedEmbeddingMethod),
|
||||
):
|
||||
raise NotImplementedError(
|
||||
f"PLE pull transport does not support {type(method).__name__}"
|
||||
)
|
||||
module._offload_quant_method = method
|
||||
module._pull_params_dtype = params_dtype or torch.get_default_dtype()
|
||||
module._pull_embedding_prefix = embedding_prefix
|
||||
if not is_offload_process():
|
||||
# The GPU-rank producer line, in upstream's format plus backend=
|
||||
# (the offload process logs its own table line from upstream's
|
||||
# constructor). The table lives in the offload process's CPU memory
|
||||
# and is not pinned; rows cross via host-mapped buffers.
|
||||
logger.info(
|
||||
"Initialized PLE embedding %s: quantization_method=%s, "
|
||||
"weight_dtype=%s, weight_device=%s, pinned=%s, backend=%s",
|
||||
embedding_prefix,
|
||||
type(method).__name__,
|
||||
module.get_offload_output_dtype(module._pull_params_dtype),
|
||||
"cpu",
|
||||
False,
|
||||
PULL_BACKEND_NAME,
|
||||
)
|
||||
return module
|
||||
|
||||
# ------------------------------------------------------------- formats
|
||||
|
||||
def get_offload_output_dtype(self, default_dtype: torch.dtype) -> torch.dtype:
|
||||
"""Rows cross the transport in their table storage dtype."""
|
||||
embedding = getattr(self, "ngram_embedding", None)
|
||||
weight = getattr(embedding, "weight", None)
|
||||
if weight is not None:
|
||||
return weight.dtype
|
||||
if isinstance(self._offload_quant_method, Qwen4ExpPLEFp8EmbeddingMethod):
|
||||
return torch.float8_e4m3fn
|
||||
return getattr(self, "_pull_params_dtype", default_dtype)
|
||||
|
||||
def get_offload_output_dim(self, default_dim: int) -> int:
|
||||
return default_dim
|
||||
|
||||
def initialize_dummy_offload_metadata(self, device: torch.device) -> None:
|
||||
"""Dummy loads skip checkpoint scales; give FP8 a unit scale."""
|
||||
if isinstance(self._offload_quant_method, Qwen4ExpPLEFp8EmbeddingMethod):
|
||||
self.register_buffer(
|
||||
"_offload_weight_scale",
|
||||
torch.ones((), dtype=torch.float32, device=device),
|
||||
persistent=False,
|
||||
)
|
||||
|
||||
def dequantize_pulled(
|
||||
self, embeddings: torch.Tensor, output_dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
"""GPU-side dequantization of rows DMA'd from the offload process.
|
||||
|
||||
Same arithmetic as upstream ``Qwen4ExpPLEFp8EmbeddingMethod.dequantize``
|
||||
(and the E2 ``_dequantize_embeddings`` FP8 branch).
|
||||
"""
|
||||
method = self._offload_quant_method
|
||||
if isinstance(method, Qwen4ExpPLEUnquantizedEmbeddingMethod):
|
||||
return embeddings
|
||||
weight_scale = getattr(self, "_offload_weight_scale", None)
|
||||
if weight_scale is None:
|
||||
raise RuntimeError(
|
||||
"FP8 PLE pull transport is missing its global scale on the GPU"
|
||||
)
|
||||
if weight_scale.device != embeddings.device:
|
||||
raise RuntimeError("FP8 PLE embedding scale must be on the output device")
|
||||
return embeddings.to(output_dtype) * weight_scale.to(output_dtype)
|
||||
|
||||
# ------------------------------------------------------------- execution
|
||||
|
||||
def start_prefetch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> None:
|
||||
"""No prefetch: rows are published by the offload process."""
|
||||
return None
|
||||
|
||||
def _compute_ngram_ids_offload(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""CPU n-gram IDs, carried unchanged from the E2 offload path.
|
||||
|
||||
Differs from upstream's CPU branch in two E2 fixes: the packed layout
|
||||
is (num_reqs, max_seq_len) rather than (num_reqs, num_tokens), and
|
||||
CUDA-graph padding rows beyond ``query_start_loc[-1]`` are not written,
|
||||
so they cannot overwrite a request's final real token.
|
||||
"""
|
||||
input_ids = input_ids.reshape(-1).long()
|
||||
query_start_loc = query_start_loc.long()
|
||||
num_reqs = query_start_loc.numel() - 1
|
||||
num_tokens = input_ids.shape[0]
|
||||
if num_reqs <= 0:
|
||||
raise ValueError("PLE requires at least one request")
|
||||
max_seq_len = max(
|
||||
1,
|
||||
int((query_start_loc[1:] - query_start_loc[:-1]).max().item()),
|
||||
)
|
||||
num_valid_tokens = min(int(query_start_loc[-1].item()), num_tokens)
|
||||
|
||||
positions = torch.arange(num_tokens, device=input_ids.device, dtype=torch.int64)
|
||||
packed = torch.full(
|
||||
(num_reqs, max_seq_len),
|
||||
self.eos_token_id,
|
||||
device=input_ids.device,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1
|
||||
request_indices.clamp_(max=num_reqs - 1)
|
||||
columns = (positions - query_start_loc[request_indices]).clamp(
|
||||
0, packed.shape[1] - 1
|
||||
)
|
||||
packed[request_indices[:num_valid_tokens], columns[:num_valid_tokens]] = (
|
||||
input_ids[:num_valid_tokens]
|
||||
)
|
||||
ngram_context = ngram_context[:num_reqs].to(
|
||||
device=input_ids.device, dtype=torch.long
|
||||
)
|
||||
|
||||
context = torch.cat([ngram_context, packed], dim=-1)
|
||||
positions_2d, position_in_segment = self._shift_precompute(
|
||||
context, self.eos_token_id
|
||||
)
|
||||
shifted = [context]
|
||||
for shift in range(1, self.ngram_size):
|
||||
shifted.append(
|
||||
self._shift_apply(
|
||||
context,
|
||||
positions_2d,
|
||||
position_in_segment,
|
||||
shift,
|
||||
self.eos_token_id,
|
||||
)
|
||||
)
|
||||
adjusted_columns = columns + self.ngram_size - 1
|
||||
id_blocks = []
|
||||
for ngram in range(2, self.ngram_size + 1):
|
||||
start = (ngram - 2) * self.heads_per_ngram
|
||||
end = start + self.heads_per_ngram
|
||||
mixed = shifted[0] * self.layer_multipliers[0]
|
||||
for index in range(1, ngram):
|
||||
mixed = torch.bitwise_xor(
|
||||
mixed, shifted[index] * self.layer_multipliers[index]
|
||||
)
|
||||
sizes = self.ngram_heads_vocab_sizes[start:end]
|
||||
offsets = self.ngram_heads_offsets[start:end]
|
||||
ids = torch.remainder(mixed.unsqueeze(-1), sizes) + offsets
|
||||
id_blocks.append(ids[request_indices, adjusted_columns])
|
||||
return torch.cat(id_blocks, dim=-1)
|
||||
|
||||
def forward_impl( # type: ignore[override]
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
output_buffer: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Offload process only: gather raw table rows for these tokens."""
|
||||
del hidden_states
|
||||
if not is_offload_process():
|
||||
raise RuntimeError(
|
||||
"PLE pull transport: forward_impl ran outside the offload "
|
||||
"process -- the GPU placeholder was never bound by the connector"
|
||||
)
|
||||
ngram_ids = self._compute_ngram_ids_offload(
|
||||
input_ids, query_start_loc, ngram_context
|
||||
)
|
||||
num_tokens = input_ids.reshape(-1).shape[0]
|
||||
weight = self.ngram_embedding.weight
|
||||
if output_buffer is not None:
|
||||
output = output_buffer[:num_tokens, : self.embedding_dim]
|
||||
torch.index_select(
|
||||
weight,
|
||||
0,
|
||||
ngram_ids.reshape(-1),
|
||||
out=output.reshape(-1, self.head_dim),
|
||||
)
|
||||
return output
|
||||
return torch.index_select(weight, 0, ngram_ids.reshape(-1)).reshape(
|
||||
num_tokens, self.embedding_dim
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------- loading
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Offload process: upstream loading. GPU worker: keep the FP8 scale.
|
||||
|
||||
The GPU placeholder owns no table; every other checkpoint tensor for
|
||||
this subtree is consumed and dropped here (the offload process loads
|
||||
the table itself). Prefix grouping may call this more than once.
|
||||
"""
|
||||
if is_offload_process():
|
||||
return Qwen4ExpNGramEmbedding.load_weights(self, weights)
|
||||
retained: set[str] = set()
|
||||
fp8 = isinstance(self._offload_quant_method, Qwen4ExpPLEFp8EmbeddingMethod)
|
||||
for name, loaded_weight in weights:
|
||||
if fp8 and name == "ngram_embedding.weight_scale":
|
||||
self.register_buffer(
|
||||
"_offload_weight_scale",
|
||||
loaded_weight.reshape(()).to(
|
||||
device=torch.accelerator.current_accelerator(),
|
||||
dtype=torch.float32,
|
||||
),
|
||||
persistent=False,
|
||||
)
|
||||
retained.add(name)
|
||||
return retained
|
||||
|
||||
|
||||
__all__ = ["PULL_BACKEND_NAME", "Qwen4ExpNGramEmbeddingPull"]
|
||||
@@ -4,7 +4,14 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Iterable
|
||||
import atexit
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Collection, Mapping
|
||||
from pathlib import Path
|
||||
from typing import ClassVar, cast
|
||||
|
||||
import torch
|
||||
@@ -16,6 +23,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.attention import (
|
||||
set_default_quant_scales,
|
||||
)
|
||||
@@ -33,6 +41,10 @@ from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
@@ -58,6 +70,260 @@ from ..common.qsa_cache import QSAForwardMetadata
|
||||
from . import model
|
||||
from .indexer_qsa import QSAIndexer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# --- FP8 K/V scale calibration collector -----------------------
|
||||
# Active ONLY when VLLM_QSA_KV_COLLECT names a writable directory. Records a
|
||||
# GPU-side running absmax of the pre-quantization BF16 K/V per layer and dumps
|
||||
# per-rank JSON (periodically + at exit) for an offline max-merge into the
|
||||
# VLLM_QSA_KV_SCALES sidecar. Calibration must run with --enforce-eager: the
|
||||
# collector mutates module-external state per step, which cudagraph capture
|
||||
# would freeze and torch.compile may not retrace. With the env unset this is a
|
||||
# constant-false branch and prod behaviour is unchanged.
|
||||
_QSA_KV_COLLECT_ENV = "VLLM_QSA_KV_COLLECT"
|
||||
_qsa_collect_dir = os.environ.get(_QSA_KV_COLLECT_ENV, "").strip()
|
||||
_qsa_collect_state: dict[str, list[torch.Tensor]] = {}
|
||||
_qsa_collect_calls = 0
|
||||
|
||||
|
||||
_qsa_collect_ident: str | None = None
|
||||
|
||||
|
||||
def _qsa_collect_identity() -> str:
|
||||
"""Stable per-process dump name: (pp, tp) rank, resolved once and cached so
|
||||
a teardown-time lookup failure cannot move the final flush to a PID-named
|
||||
file. TP-only runs keep the historical name."""
|
||||
global _qsa_collect_ident
|
||||
if _qsa_collect_ident is not None:
|
||||
return _qsa_collect_ident
|
||||
try:
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_rank
|
||||
|
||||
rank = get_tensor_model_parallel_rank()
|
||||
pp_group = get_pp_group()
|
||||
pp_tag = f"pp{pp_group.rank_in_group}_" if pp_group.world_size > 1 else ""
|
||||
_qsa_collect_ident = f"{pp_tag}rank{rank}"
|
||||
except Exception:
|
||||
return f"pid{os.getpid()}" # not cached: retry once the groups exist
|
||||
return _qsa_collect_ident
|
||||
|
||||
|
||||
def _qsa_collect_dump() -> None:
|
||||
if not _qsa_collect_state:
|
||||
return
|
||||
out = {
|
||||
name: {"k_absmax": float(st[0].item()), "v_absmax": float(st[1].item())}
|
||||
for name, st in _qsa_collect_state.items()
|
||||
}
|
||||
path = Path(_qsa_collect_dir) / f"qsa_absmax_{_qsa_collect_identity()}.json"
|
||||
tmp = path.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(out, indent=1), encoding="utf-8")
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def _qsa_collect_absmax(
|
||||
layer_name: str, key: torch.Tensor, value: torch.Tensor
|
||||
) -> None:
|
||||
global _qsa_collect_calls
|
||||
k_abs = key.detach().abs().amax().float()
|
||||
v_abs = value.detach().abs().amax().float()
|
||||
st = _qsa_collect_state.get(layer_name)
|
||||
if st is None:
|
||||
if not _qsa_collect_state:
|
||||
atexit.register(_qsa_collect_dump)
|
||||
_qsa_collect_state[layer_name] = [k_abs, v_abs]
|
||||
else:
|
||||
torch.maximum(st[0], k_abs, out=st[0])
|
||||
torch.maximum(st[1], v_abs, out=st[1])
|
||||
_qsa_collect_calls += 1
|
||||
if _qsa_collect_calls % 2000 == 0:
|
||||
_qsa_collect_dump()
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# --- FP8 K/V runtime clipping counter --------------------------
|
||||
# Under-coverage guard for the calibrated scales sidecar: counts K/V elements
|
||||
# that would saturate E4M3 after scaling (|x| > 448 * scale), i.e. inputs that
|
||||
# exceed the calibration-time absmax. Active ONLY when VLLM_QSA_KV_CLIP_COUNT
|
||||
# is set to a positive read interval in seconds.
|
||||
#
|
||||
# Graph-safety: unlike the calibration collector above, this MUST work under
|
||||
# FULL cudagraph replay, where the Python forward does not run per step. The
|
||||
# counters are persistent per-layer GPU int64 tensors allocated on the first
|
||||
# (eager, pre-capture) forward; the increment is pure captured GPU ops, so
|
||||
# graph replay keeps counting. A daemon reader thread .item()s the counters on
|
||||
# a cadence and logs: WARNING on any new clips, INFO heartbeat every 10th pass
|
||||
# so a quiet log still proves the instrument is alive.
|
||||
#
|
||||
# Since v2.5 the reader thread is started AFTER warmup and cudagraph capture
|
||||
# (guard_telemetry.register_post_capture_hook, run by start_reader() from
|
||||
# gpu_worker), never from the first eager forward. Before, it started inside the
|
||||
# first forward, i.e. before capture, and its device-to-host copies could run on
|
||||
# a second thread while a capture was active -- the call class
|
||||
# cudaStreamCaptureModeGlobal rejects. That is why COUNTERS shipped off.
|
||||
_QSA_CLIP_ENV = "VLLM_QSA_KV_CLIP_COUNT"
|
||||
_qsa_clip_raw = os.environ.get(_QSA_CLIP_ENV, "").strip()
|
||||
try:
|
||||
_qsa_clip_interval = float(_qsa_clip_raw) if _qsa_clip_raw else 0.0
|
||||
except ValueError as exc: # fail closed: a set-but-broken gate must not be silent
|
||||
raise ValueError(
|
||||
f"{_QSA_CLIP_ENV} must be a read interval in seconds, got {_qsa_clip_raw!r}"
|
||||
) from exc
|
||||
if _qsa_clip_raw and not (math.isfinite(_qsa_clip_interval) and _qsa_clip_interval > 0.0):
|
||||
raise ValueError(
|
||||
f"{_QSA_CLIP_ENV} must be a finite positive number of seconds, got {_qsa_clip_raw!r}"
|
||||
)
|
||||
_qsa_clip_counters: dict[str, torch.Tensor] = {}
|
||||
# v2 reader fix: the reader must never block on the compute
|
||||
# stream; see guard_telemetry.NonBlockingReader.
|
||||
from vllm.v1.worker.guard_telemetry import ( # noqa: E402
|
||||
NonBlockingReader as _NonBlockingReader,
|
||||
)
|
||||
|
||||
_qsa_clip_nb = _NonBlockingReader()
|
||||
_qsa_clip_reader_started = False
|
||||
_qsa_clip_lock = threading.Lock()
|
||||
|
||||
|
||||
def _qsa_clip_reader() -> None:
|
||||
last: dict[str, tuple[int, int]] = {}
|
||||
passes = 0
|
||||
while True:
|
||||
time.sleep(_qsa_clip_interval)
|
||||
passes += 1
|
||||
try:
|
||||
totals_k = totals_v = 0
|
||||
for name, ctr in list(_qsa_clip_counters.items()):
|
||||
if not _qsa_clip_nb.registered(name):
|
||||
_qsa_clip_nb.register(name, ctr) # reader thread: never capturing
|
||||
ready = _qsa_clip_nb.poll()
|
||||
for name, ctr in list(_qsa_clip_counters.items()):
|
||||
row = ready.get(name)
|
||||
if row is None:
|
||||
# copy still in flight: report on the next pass
|
||||
prev_k, prev_v = last.get(name, (0, 0))
|
||||
totals_k += prev_k
|
||||
totals_v += prev_v
|
||||
continue
|
||||
k_total = int(row[0])
|
||||
v_total = int(row[1])
|
||||
totals_k += k_total
|
||||
totals_v += v_total
|
||||
prev_k, prev_v = last.get(name, (0, 0))
|
||||
if k_total > prev_k or v_total > prev_v:
|
||||
logger.warning(
|
||||
"QSA KV CLIPPING on %s: k +%d (total %d), v +%d (total %d)"
|
||||
" -- inputs exceeded the calibration absmax; the scales"
|
||||
" sidecar may need re-calibration",
|
||||
name,
|
||||
k_total - prev_k,
|
||||
k_total,
|
||||
v_total - prev_v,
|
||||
v_total,
|
||||
)
|
||||
last[name] = (k_total, v_total)
|
||||
if passes % 10 == 1:
|
||||
logger.info(
|
||||
"QSA clip counters alive: %d layer(s), totals k=%d v=%d",
|
||||
len(_qsa_clip_counters),
|
||||
totals_k,
|
||||
totals_v,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("QSA clip reader pass failed; thread continues")
|
||||
|
||||
|
||||
def _qsa_clip_start_reader() -> None:
|
||||
"""Start the reader thread. Runs after warmup + capture (post-capture hook); idempotent."""
|
||||
global _qsa_clip_reader_started
|
||||
with _qsa_clip_lock:
|
||||
if _qsa_clip_reader_started:
|
||||
return
|
||||
_qsa_clip_reader_started = True
|
||||
threading.Thread(
|
||||
target=_qsa_clip_reader,
|
||||
name="qsa-clip-reader",
|
||||
daemon=True,
|
||||
).start()
|
||||
# The leading "QSA clip counter armed (read interval Ns)" is the token the serving smoke greps
|
||||
# (v2.2.0 logged it from the first forward); keep it verbatim when editing this line.
|
||||
logger.info(
|
||||
"QSA clip counter armed (read interval %.0fs); reader started after capture "
|
||||
"(%d layer(s) registered)",
|
||||
_qsa_clip_interval,
|
||||
len(_qsa_clip_counters),
|
||||
)
|
||||
|
||||
|
||||
if _qsa_clip_interval > 0.0:
|
||||
from vllm.v1.worker.guard_telemetry import register_post_capture_hook # noqa: E402
|
||||
|
||||
register_post_capture_hook(_qsa_clip_start_reader)
|
||||
|
||||
|
||||
def _qsa_clip_count(
|
||||
layer: Qwen4ExpQSAAttention,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
slot_mapping: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
ctr = _qsa_clip_counters.get(layer.layer_name)
|
||||
if ctr is None:
|
||||
# First forward is eager (profiling/warmup precede capture), so the
|
||||
# allocation never happens inside cudagraph capture. The READER is not
|
||||
# started here: _qsa_clip_start_reader runs after capture.
|
||||
ctr = torch.zeros(2, dtype=torch.int64, device=key.device)
|
||||
_qsa_clip_counters[layer.layer_name] = ctr # registered with the reader by the reader thread (never during capture)
|
||||
# Thresholds are 1-D so the comparison promotes to float32; a 0-dim float32
|
||||
# tensor promotes like a Python scalar and the threshold would be rounded to
|
||||
# the BF16 grid of K/V, undercounting values just above the calibrated ceiling.
|
||||
k_ceiling = (448.0 * layer._k_scale).to(torch.float32).view(1)
|
||||
v_ceiling = (448.0 * layer._v_scale).to(torch.float32).view(1)
|
||||
k_over = key.detach().abs() > k_ceiling
|
||||
v_over = value.detach().abs() > v_ceiling
|
||||
if slot_mapping is not None:
|
||||
# The CUDA writer (reshape_and_cache_flash) skips rows with a negative
|
||||
# slot: padding rows of a captured batch size and, under the V2 model
|
||||
# runner, stale rows of the persistent input buffers. Count only what
|
||||
# is actually written, with fixed-shape masking so the op is capturable.
|
||||
n = min(slot_mapping.shape[0], key.shape[0])
|
||||
written = (slot_mapping[:n] >= 0).view(-1, 1, 1)
|
||||
k_over = k_over[:n] & written
|
||||
v_over = v_over[:n] & written
|
||||
ctr[0] += k_over.sum()
|
||||
ctr[1] += v_over.sum()
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
_QSA_FP8_CACHE_DTYPES = ("fp8", "fp8_e4m3")
|
||||
_QSA_SUPPORTED_CACHE_DTYPES = ("auto", "bfloat16", *_QSA_FP8_CACHE_DTYPES)
|
||||
|
||||
|
||||
def _is_qsa_fp8_cache_dtype(cache_dtype: str) -> bool:
|
||||
return cache_dtype in _QSA_FP8_CACHE_DTYPES
|
||||
|
||||
|
||||
def _validated_qsa_fp8_dtype(cache_dtype: str) -> torch.dtype | None:
|
||||
"""Return the platform E4M3 dtype after enforcing the validated HW gate."""
|
||||
|
||||
if not _is_qsa_fp8_cache_dtype(cache_dtype):
|
||||
return None
|
||||
if not current_platform.is_cuda():
|
||||
raise ValueError(
|
||||
"Qwen4Exp QSA E4M3 KV cache is supported only on CUDA; "
|
||||
"ROCm and other platforms have not been validated"
|
||||
)
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability.to_int() != 86:
|
||||
cap_str = (
|
||||
capability.as_version_str() if capability is not None else "unknown"
|
||||
)
|
||||
raise ValueError(
|
||||
"Qwen4Exp QSA E4M3 KV cache is validated only on SM86, but "
|
||||
f"{current_platform.get_device_name()} has compute capability "
|
||||
f"{cap_str}. Re-run with --kv-cache-dtype bfloat16."
|
||||
)
|
||||
return current_platform.fp8_dtype()
|
||||
|
||||
|
||||
class Qwen4ExpQSAQKVIndexerLinear(MergedColumnParallelLinear):
|
||||
"""Pack sharded Q/gate, K, V and replicated indexer Q/K in one GEMM."""
|
||||
@@ -135,10 +401,6 @@ class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend):
|
||||
"""FullAttentionSpec backend used by the merged QSA owner."""
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||
# fp8/fp8_e4m3: e4m3 bytes in a uint8 cache, written by reshape_and_cache
|
||||
# with the layer's per-tensor scales and dequantized on load inside the QSA
|
||||
# Triton kernel. flash-attn never runs over this cache, so its fp8 probe
|
||||
# does not apply (see supports_kv_cache_dtype and the impl constructor).
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"auto",
|
||||
"bfloat16",
|
||||
@@ -148,6 +410,12 @@ class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend):
|
||||
|
||||
@classmethod
|
||||
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
|
||||
# The parent additionally probes flash-attn for quantized-KV support
|
||||
# and rejects fp8 wherever FA has no kernel for it (all of sm_86).
|
||||
# QSA never runs flash-attn over this cache: reshape_and_cache_flash
|
||||
# writes the e4m3 bytes and our Triton kernel dequantizes them on load,
|
||||
# so the class list alone decides. See the impl constructor for the
|
||||
# matching "auto"-then-restore of the same probe.
|
||||
return kv_cache_dtype is None or kv_cache_dtype in cls.supported_kv_cache_dtypes
|
||||
|
||||
@classmethod
|
||||
@@ -163,10 +431,36 @@ class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend):
|
||||
use_mm_prefix: bool,
|
||||
device_capability: DeviceCapability,
|
||||
) -> str | None:
|
||||
# QSA dequantizes the fp8 KV in its own Triton kernel and never runs
|
||||
# flash-attn over the quantized cache, so the parent's fp8-KV rejection
|
||||
# does not apply and every combination it is handed is accepted here.
|
||||
return None
|
||||
# Suppress ONLY the parent's fp8 clause ("FP8 KV cache requires FA3 on
|
||||
# SM90 or FA4 on SM100"), which is about flash-attn's own kernels and
|
||||
# so does not apply to a cache QSA reads in Triton. The real device
|
||||
# gate for our fp8 path is `_validated_qsa_fp8_dtype`, which raises on
|
||||
# any capability != 86.
|
||||
#
|
||||
# Everything else the parent rejects still holds and is returned
|
||||
# unchanged: the sink/compute-capability clause and the
|
||||
# mm-prefix/FA-version clause. Returning None for every combination
|
||||
# would advertise fewer restrictions than `forward_qsa` enforces (it
|
||||
# raises on ALiBi, sinks and sliding windows).
|
||||
#
|
||||
# Same shape as the impl constructor: hide the fp8 dtype from the
|
||||
# probe, keep the rest of the parent's behaviour.
|
||||
probe_kv_cache_dtype = kv_cache_dtype
|
||||
if probe_kv_cache_dtype is not None and _is_qsa_fp8_cache_dtype(
|
||||
probe_kv_cache_dtype
|
||||
):
|
||||
probe_kv_cache_dtype = None
|
||||
return super().supports_combination(
|
||||
head_size,
|
||||
dtype,
|
||||
probe_kv_cache_dtype,
|
||||
block_size,
|
||||
use_mla,
|
||||
has_sink,
|
||||
use_sparse,
|
||||
use_mm_prefix,
|
||||
device_capability,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
@@ -214,14 +508,18 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
||||
kv_sharing_target_layer_name: str | None = None,
|
||||
sinks: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
# The parent constructor probes flash-attn for quantized-KV support and
|
||||
# raises where it is unavailable (sm120), but QSA dequantizes fp8 inside
|
||||
# its own Triton kernel and never runs flash-attn over the cache. Hand
|
||||
# the parent "auto" for that probe and restore the real dtype afterwards:
|
||||
# the parent only uses it there, and do_kv_cache_update reads the
|
||||
# attribute at call time.
|
||||
# QSA never dispatches to a FlashAttention kernel: attention runs through
|
||||
# forward_qsa -> qsa_sparse_paged_attention (our Triton sparse-GQA kernel)
|
||||
# and the cache write goes through the CUDA reshape_and_cache_flash op.
|
||||
# FlashAttentionImpl.__init__ nevertheless rejects any quantized KV dtype
|
||||
# that FA itself cannot read, and sm_86 has no FP8 FA kernel. Present
|
||||
# "auto" to the base class, then restore the real dtype. The only other
|
||||
# kv_cache_dtype use in that constructor is an SM90/FA4 dequant path that
|
||||
# cannot trigger here, and supports_quant_query_input is set below anyway.
|
||||
# The parameters are spelled out rather than forwarded as *args/**kwargs
|
||||
# so the substitution names the parameter instead of indexing position 6.
|
||||
real_kv_cache_dtype = kv_cache_dtype
|
||||
if kv_cache_dtype in ("fp8", "fp8_e4m3"):
|
||||
if _is_qsa_fp8_cache_dtype(kv_cache_dtype):
|
||||
kv_cache_dtype = "auto"
|
||||
super().__init__(
|
||||
num_heads,
|
||||
@@ -243,10 +541,12 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA does not support decode context parallelism"
|
||||
)
|
||||
if self.kv_cache_dtype not in ("auto", "bfloat16", "fp8", "fp8_e4m3"):
|
||||
if self.kv_cache_dtype not in _QSA_SUPPORTED_CACHE_DTYPES:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA requires a BF16 or FP8-e4m3 main KV cache"
|
||||
"Qwen4Exp QSA supports BF16 or static per-tensor E4M3 main "
|
||||
f"KV caches, not {self.kv_cache_dtype!r}"
|
||||
)
|
||||
self.qsa_fp8_dtype = _validated_qsa_fp8_dtype(self.kv_cache_dtype)
|
||||
self.supports_quant_query_input = False
|
||||
|
||||
def forward_qsa(
|
||||
@@ -283,26 +583,42 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
||||
logical_indices = topk_buffer[:num_tokens]
|
||||
token_to_req = token_to_req[:num_tokens]
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
k_scale = v_scale = None
|
||||
if self.kv_cache_dtype in ("fp8", "fp8_e4m3"):
|
||||
# The cache is allocated as uint8; reinterpret the e4m3 bytes
|
||||
# (same itemsize, so shape and strides are preserved).
|
||||
key_cache = key_cache.view(torch.float8_e4m3fn)
|
||||
value_cache = value_cache.view(torch.float8_e4m3fn)
|
||||
# Host-side per-tensor dequant scales (Python floats), as used by
|
||||
# other host-scale backends; folded into the kernel's scales.
|
||||
k_scale = layer._k_scale_float
|
||||
v_scale = layer._v_scale_float
|
||||
if query.dtype != torch.bfloat16 or key_cache.dtype not in (
|
||||
torch.bfloat16,
|
||||
torch.float8_e4m3fn,
|
||||
):
|
||||
if query.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("Qwen4Exp QSA requires a BF16 query")
|
||||
if key_cache.dtype != value_cache.dtype:
|
||||
raise TypeError("Qwen4Exp QSA K and V cache dtypes must match")
|
||||
if _is_qsa_fp8_cache_dtype(self.kv_cache_dtype):
|
||||
if self.qsa_fp8_dtype is None:
|
||||
raise RuntimeError("QSA FP8 dtype was not initialized")
|
||||
if key_cache.dtype not in (torch.uint8, self.qsa_fp8_dtype):
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA E4M3 cache storage must use raw uint8 or "
|
||||
f"the platform FP8 dtype, not {key_cache.dtype}"
|
||||
)
|
||||
elif key_cache.dtype != torch.bfloat16:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA requires BF16 Q and BF16 or FP8-e4m3 K/V"
|
||||
"Qwen4Exp QSA BF16 mode requires a BF16 main KV cache"
|
||||
)
|
||||
|
||||
from .ops.qsa import qsa_sparse_paged_attention
|
||||
|
||||
# Announce the effective scales once per layer. load_qsa_static_kv_scales()
|
||||
# is an explicit integration call, so a deployment that forgets it runs at
|
||||
# the vLLM default of 1.0 -- which is a legitimate configuration but must
|
||||
# never be a silent one. This makes the actual values greppable per rank.
|
||||
if _is_qsa_fp8_cache_dtype(self.kv_cache_dtype) and not getattr(
|
||||
self, "_qsa_scales_logged", False
|
||||
):
|
||||
self._qsa_scales_logged = True
|
||||
logger.info(
|
||||
"Qwen4Exp QSA E4M3 KV active on %s: k_scale=%.6g v_scale=%.6g%s",
|
||||
getattr(layer, "layer_name", "<unnamed>"),
|
||||
float(layer._k_scale),
|
||||
float(layer._v_scale),
|
||||
"" if float(layer._k_scale) != 1.0 or float(layer._v_scale) != 1.0
|
||||
else " (DEFAULT 1.0 -- no static calibration loaded)",
|
||||
)
|
||||
|
||||
qsa_sparse_paged_attention(
|
||||
query[:num_tokens],
|
||||
key_cache,
|
||||
@@ -311,9 +627,9 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
||||
attn_metadata.block_table,
|
||||
token_to_req,
|
||||
use_prefill_config,
|
||||
output[:num_tokens],
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
out=output[:num_tokens],
|
||||
k_scale=layer._k_scale,
|
||||
v_scale=layer._v_scale,
|
||||
output_gate=output_gate[:num_tokens],
|
||||
)
|
||||
return output
|
||||
@@ -341,17 +657,20 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
raise ValueError("Qwen4Exp QSA requires a paged KV cache")
|
||||
if model_config.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
|
||||
if cache_config.cache_dtype not in (
|
||||
"auto",
|
||||
"bfloat16",
|
||||
"fp8",
|
||||
"fp8_e4m3",
|
||||
):
|
||||
if cache_config.cache_dtype not in _QSA_SUPPORTED_CACHE_DTYPES:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA requires a BF16 or FP8-e4m3 main KV cache"
|
||||
"Qwen4Exp QSA supports BF16 or static per-tensor E4M3 main "
|
||||
f"KV caches, not {cache_config.cache_dtype!r}"
|
||||
)
|
||||
qsa_fp8_dtype = _validated_qsa_fp8_dtype(cache_config.cache_dtype)
|
||||
kv_cache_scheme = getattr(quant_config, "kv_cache_scheme", None)
|
||||
if kv_cache_scheme is not None:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA does not implement kv_cache_scheme variants. "
|
||||
"Dynamic, per-head, per-token, and other non-scalar KV scales "
|
||||
"are unsupported; use static scalar per-layer K/V scales from "
|
||||
"load_qsa_static_kv_scales()."
|
||||
)
|
||||
if getattr(quant_config, "kv_cache_scheme", None) is not None:
|
||||
raise NotImplementedError("Qwen4Exp QSA does not support KV quantization")
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if (
|
||||
parallel_config.prefill_context_parallel_size > 1
|
||||
@@ -448,12 +767,23 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
|
||||
self.kv_cache_dtype, model_config
|
||||
)
|
||||
if self.kv_cache_torch_dtype not in (torch.bfloat16, torch.uint8):
|
||||
if _is_qsa_fp8_cache_dtype(self.kv_cache_dtype):
|
||||
if qsa_fp8_dtype is None:
|
||||
raise RuntimeError("QSA FP8 dtype was not initialized")
|
||||
if self.kv_cache_torch_dtype not in (torch.uint8, qsa_fp8_dtype):
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA E4M3 cache storage resolved to unsupported "
|
||||
f"dtype {self.kv_cache_torch_dtype}"
|
||||
)
|
||||
elif self.kv_cache_torch_dtype != torch.bfloat16:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA requires BF16 or FP8-e4m3 (uint8) cache storage"
|
||||
"Qwen4Exp QSA BF16 mode requires BF16 cache storage"
|
||||
)
|
||||
self.kv_sharing_target_layer_name = None
|
||||
self.kv_cache = torch.tensor([])
|
||||
# The checkpoint has no KV scale scheme. These registered scalar
|
||||
# buffers therefore remain 1.0 unless explicitly replaced after
|
||||
# weight loading by load_qsa_static_kv_scales().
|
||||
set_default_quant_scales(self, register_buffer=True)
|
||||
|
||||
self.attn_backend = Qwen4ExpQSAFlashAttentionBackend
|
||||
@@ -578,6 +908,10 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
)
|
||||
if selected.shape != (num_tokens, self.indexer.packed_output_width):
|
||||
raise RuntimeError("QSA indexer returned an invalid selection shape")
|
||||
if _qsa_clip_interval > 0.0 and _is_qsa_fp8_cache_dtype(
|
||||
self.kv_cache_dtype
|
||||
):
|
||||
_qsa_clip_count(self, key, value, main_metadata.slot_mapping)
|
||||
impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl)
|
||||
if main_outputs is None:
|
||||
assert key is not None and value is not None
|
||||
@@ -617,34 +951,235 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
else:
|
||||
projected_qk, _ = self.indexer.index_qk_proj(hidden_states)
|
||||
num_tokens = hidden_states.shape[0]
|
||||
if not self.use_fused_qsa_prepare:
|
||||
q, k, v, gate = self._project_qkv_gate(qkv, positions)
|
||||
assert gate is not None
|
||||
query = q.view(num_tokens, self.num_heads, self.head_dim)
|
||||
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
|
||||
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
|
||||
query = q.view(num_tokens, self.num_heads, self.head_dim)
|
||||
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
|
||||
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
|
||||
if _qsa_collect_dir:
|
||||
_qsa_collect_absmax(self.layer_name, key, value)
|
||||
attn_output = torch.empty_like(query)
|
||||
# Keep the index projection outside the eager break / splitting op.
|
||||
projected_qk, _ = self.indexer.index_qk_proj(hidden_states)
|
||||
# Compile-profile guard: the QSA transaction reads per-step attention metadata from
|
||||
# the forward context, so under VLLM_COMPILE it must be opaque to
|
||||
# Dynamo and a piecewise split point (compilation.py _attention_ops).
|
||||
# Restores the op #55272 removed, with #55309's fused output gate.
|
||||
encoded_layer_name = _encode_layer_name(self.layer_name)
|
||||
if current_platform.opaque_attention_op():
|
||||
torch.ops.vllm.qwen4_exp_qsa_with_output(
|
||||
projected_qk,
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_output,
|
||||
gate,
|
||||
encoded_layer_name,
|
||||
)
|
||||
else:
|
||||
# Norm/RoPE/gate and the K/V cache write happen inside _run_qsa.
|
||||
query = key = value = gate = None
|
||||
attn_output = qkv.new_empty(num_tokens, self.num_heads, self.head_dim)
|
||||
self._run_qsa(
|
||||
projected_qk,
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_output,
|
||||
gate,
|
||||
qkv,
|
||||
)
|
||||
qwen4_exp_qsa_with_output(
|
||||
projected_qk,
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_output,
|
||||
gate,
|
||||
encoded_layer_name,
|
||||
)
|
||||
flat_output = attn_output.view(num_tokens, -1)
|
||||
output, _ = self.o_proj(flat_output)
|
||||
return output
|
||||
|
||||
|
||||
def load_qsa_static_kv_scales(
|
||||
layers: nn.Module | Mapping[str, nn.Module],
|
||||
sidecar_path: str | Path,
|
||||
*,
|
||||
strict: bool = True,
|
||||
expected_layer_names: Collection[str] | None = None,
|
||||
) -> list[str]:
|
||||
"""Load static scalar QSA K/V scales into already-loaded model layers.
|
||||
|
||||
Call this on every model worker after checkpoint weight loading finishes
|
||||
and before any profiling, warmup, or cudagraph capture. ``layers`` may be
|
||||
the loaded root module or ``compilation_config.static_forward_context``.
|
||||
|
||||
The JSON schema is::
|
||||
|
||||
{
|
||||
"model.layers.3.self_attn.attn": {
|
||||
"k_scale": 0.25,
|
||||
"v_scale": 2.0
|
||||
}
|
||||
}
|
||||
|
||||
``strict=False`` permits a deliberate subset; omitted layers keep the
|
||||
vLLM default scale of 1.0. Unknown names and non-scalar values are always
|
||||
errors.
|
||||
|
||||
Pipeline parallelism: a rank holds only its stage's layers, so the sidecar
|
||||
legitimately names layers this rank does not own. Pass
|
||||
``expected_layer_names`` = the QSA layer names of the WHOLE model; names are
|
||||
validated against that set, ``strict`` requires the sidecar to cover all
|
||||
of it (checkable on every rank), and the scales are applied to the local
|
||||
intersection only. Without it (TP-only callers) the local layers are the
|
||||
universe, as before.
|
||||
"""
|
||||
|
||||
path = Path(sidecar_path)
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError(f"{path} must contain a JSON object keyed by layer name")
|
||||
|
||||
if isinstance(layers, nn.Module):
|
||||
qsa_layers = {
|
||||
layer.layer_name: layer
|
||||
for layer in layers.modules()
|
||||
if isinstance(layer, Qwen4ExpQSAAttention)
|
||||
}
|
||||
else:
|
||||
qsa_layers = {
|
||||
name: layer
|
||||
for name, layer in layers.items()
|
||||
if isinstance(name, str)
|
||||
and isinstance(layer, Qwen4ExpQSAAttention)
|
||||
}
|
||||
|
||||
if not qsa_layers and expected_layer_names is None:
|
||||
raise ValueError("No Qwen4Exp QSA attention layers were supplied")
|
||||
non_fp8_layers = sorted(
|
||||
name
|
||||
for name, layer in qsa_layers.items()
|
||||
if not _is_qsa_fp8_cache_dtype(layer.kv_cache_dtype)
|
||||
)
|
||||
if non_fp8_layers:
|
||||
raise ValueError(
|
||||
"Static QSA KV scales may only be injected into E4M3 layers; "
|
||||
f"non-FP8 layers: {non_fp8_layers}"
|
||||
)
|
||||
|
||||
if expected_layer_names is None:
|
||||
universe = set(qsa_layers)
|
||||
else:
|
||||
universe = set(expected_layer_names)
|
||||
not_expected = sorted(name for name in qsa_layers if name not in universe)
|
||||
if not_expected:
|
||||
raise ValueError(
|
||||
"Local QSA layers outside expected_layer_names: "
|
||||
f"{not_expected}"
|
||||
)
|
||||
unknown = sorted(name for name in raw if name not in universe)
|
||||
if unknown:
|
||||
raise ValueError(f"Unknown QSA layer names in {path}: {unknown}")
|
||||
missing = sorted(name for name in universe if name not in raw)
|
||||
if strict and missing:
|
||||
raise ValueError(f"Missing QSA layer scales in {path}: {missing}")
|
||||
remote = sorted(name for name in raw if name not in qsa_layers)
|
||||
if remote:
|
||||
logger.info(
|
||||
"Qwen4Exp QSA: %d sidecar layer(s) belong to other pipeline stages "
|
||||
"and are skipped on this rank: %s%s",
|
||||
len(remote),
|
||||
", ".join(remote[:4]),
|
||||
" ..." if len(remote) > 4 else "",
|
||||
)
|
||||
|
||||
validated: dict[str, tuple[float, float]] = {}
|
||||
for layer_name, entry in raw.items():
|
||||
if not isinstance(entry, dict):
|
||||
raise ValueError(f"{layer_name} scales must be a JSON object")
|
||||
if set(entry) != {"k_scale", "v_scale"}:
|
||||
raise ValueError(
|
||||
f"{layer_name} must contain exactly k_scale and v_scale"
|
||||
)
|
||||
|
||||
values: list[float] = []
|
||||
for field in ("k_scale", "v_scale"):
|
||||
value = entry[field]
|
||||
if type(value) not in (int, float):
|
||||
raise ValueError(
|
||||
f"{layer_name}.{field} must be a scalar JSON number"
|
||||
)
|
||||
scale = float(value)
|
||||
if not math.isfinite(scale) or scale <= 0.0:
|
||||
raise ValueError(
|
||||
f"{layer_name}.{field} must be finite and positive"
|
||||
)
|
||||
values.append(scale)
|
||||
|
||||
layer = qsa_layers.get(layer_name)
|
||||
if layer is None:
|
||||
continue # another pipeline stage's layer: schema-checked only
|
||||
for field in ("_k_scale", "_v_scale"):
|
||||
tensor = getattr(layer, field, None)
|
||||
if not isinstance(tensor, torch.Tensor) or tensor.numel() != 1:
|
||||
raise ValueError(
|
||||
f"{layer_name}.{field} must be an existing scalar tensor"
|
||||
)
|
||||
validated[layer_name] = (values[0], values[1])
|
||||
|
||||
with torch.no_grad():
|
||||
for layer_name, (k_scale, v_scale) in validated.items():
|
||||
layer = qsa_layers[layer_name]
|
||||
layer._k_scale.fill_(k_scale)
|
||||
layer._v_scale.fill_(v_scale)
|
||||
|
||||
return sorted(validated)
|
||||
|
||||
|
||||
def qwen4_exp_qsa_with_output(
|
||||
projected_qk: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
output_gate: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
"""Run the complete QSA state/update/attend transaction."""
|
||||
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
if not isinstance(layer, Qwen4ExpQSAAttention):
|
||||
raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner")
|
||||
layer._run_qsa(
|
||||
projected_qk,
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
output,
|
||||
output_gate,
|
||||
)
|
||||
|
||||
|
||||
def qwen4_exp_qsa_with_output_fake(
|
||||
projected_qk: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
output_gate: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
del projected_qk, positions, query, key, value, output, output_gate, layer_name
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_qsa_with_output",
|
||||
op_func=qwen4_exp_qsa_with_output,
|
||||
mutates_args=["output"],
|
||||
fake_impl=qwen4_exp_qsa_with_output_fake,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"QSAIndexer",
|
||||
"Qwen4ExpQSAAttention",
|
||||
"Qwen4ExpQSAFlashAttentionBackend",
|
||||
"Qwen4ExpQSAFlashAttentionImpl",
|
||||
"load_qsa_static_kv_scales",
|
||||
"qwen4_exp_qsa_with_output",
|
||||
]
|
||||
|
||||
@@ -11,6 +11,12 @@
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker import guard_telemetry
|
||||
|
||||
# Bias so a negative row/index survives atomic_max on a zeroed counter
|
||||
# -- it MUST be a tl.constexpr: a plain module-level int is not
|
||||
# readable from a @triton.jit body.
|
||||
_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS)
|
||||
|
||||
from .op import exp, log
|
||||
|
||||
@@ -36,6 +42,7 @@ def fused_recurrent_gated_delta_rule_fwd_kernel(
|
||||
cu_seqlens,
|
||||
ssm_state_indices,
|
||||
num_accepted_tokens,
|
||||
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||
scale,
|
||||
N: tl.int64, # num of sequences
|
||||
T: tl.int64, # num of tokens
|
||||
@@ -106,12 +113,36 @@ def fused_recurrent_gated_delta_rule_fwd_kernel(
|
||||
i_t = tl.load(num_accepted_tokens + i_n).to(tl.int64) - 1
|
||||
else:
|
||||
i_t = 0
|
||||
# Load state index and check for invalid entries
|
||||
state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq + i_t).to(
|
||||
tl.int64
|
||||
)
|
||||
# Skip if state index is invalid (NULL_BLOCK_ID=0)
|
||||
# PR #50021: i_t is num_accepted_tokens - 1 and is otherwise
|
||||
# unbounded, while ssm_state_indices has only stride_indices_seq
|
||||
# columns per request. A zero accepted count gives i_t == -1, a
|
||||
# read before this request's row (and before the tensor for
|
||||
# i_n == 0); a stale or too-large count reads past the row. The
|
||||
# state_idx <= 0 guard below rejects only non-positive values, so
|
||||
# an out-of-range read returning a garbage positive int flowed into
|
||||
# h0 + state_idx * stride_init_state_token and faulted the SM.
|
||||
#
|
||||
# This is the SAME defect as the decode-state guard, in the twin kernel. An
|
||||
# earlier revision left it unguarded and dropped the upstream test's arm for it;
|
||||
# it is now ported rather than caveated.
|
||||
idx_in_row = (i_t >= 0) & (i_t < stride_indices_seq)
|
||||
if (i_t < 0) or (i_t >= stride_indices_seq):
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (i_n).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, (i_t).to(tl.int64) + _GUARD_BIAS)
|
||||
state_idx = tl.load(
|
||||
ssm_state_indices + i_n * stride_indices_seq + i_t,
|
||||
mask=idx_in_row,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
# Skip invalid state indices without exposing uninitialized output.
|
||||
# o is q.new_empty in the entry below, so the previous early return
|
||||
# left this request's rows holding whatever the allocator returned.
|
||||
if state_idx <= 0:
|
||||
zero = tl.zeros([BV], dtype=tl.float32).to(p_o.dtype.element_ty)
|
||||
for _ in range(0, T):
|
||||
tl.store(p_o, zero, mask=mask_v)
|
||||
p_o += HV * V
|
||||
return
|
||||
p_h0 = h0 + state_idx * stride_init_state_token
|
||||
else:
|
||||
@@ -227,6 +258,9 @@ def fused_recurrent_gated_delta_rule_fwd(
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
guard_ctr=guard_telemetry.counter(
|
||||
guard_telemetry.SITE_GDN_RECURRENT_STATE, q.device
|
||||
),
|
||||
scale=scale,
|
||||
N=N,
|
||||
T=T,
|
||||
|
||||
@@ -10,6 +10,14 @@
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker import guard_telemetry
|
||||
|
||||
# Bias so a negative row/index survives atomic_max on a zeroed counter
|
||||
# -- it MUST be a tl.constexpr: a plain module-level int is not readable
|
||||
# from a @triton.jit body -- an earlier build shipped one and every guarded kernel died
|
||||
# with "NameError: Cannot access global variable _GUARD_BIAS" on the smoke.
|
||||
_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS)
|
||||
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
@@ -37,6 +45,7 @@ def fused_sigmoid_gating_delta_rule_update_kernel(
|
||||
cu_seqlens,
|
||||
ssm_state_indices,
|
||||
num_accepted_tokens,
|
||||
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||
scale,
|
||||
N: tl.int64, # num of sequences
|
||||
T: tl.int64, # num of tokens
|
||||
@@ -106,12 +115,32 @@ def fused_sigmoid_gating_delta_rule_update_kernel(
|
||||
i_t = tl.load(num_accepted_tokens + i_n).to(tl.int64) - 1
|
||||
else:
|
||||
i_t = 0
|
||||
# Load state index and check for invalid entries
|
||||
state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq + i_t).to(
|
||||
tl.int64
|
||||
)
|
||||
# Skip if state index is invalid (NULL_BLOCK_ID=0)
|
||||
# PR #50021: i_t is num_accepted_tokens - 1 and is otherwise
|
||||
# unbounded against a stride_indices_seq-column tensor. A zero
|
||||
# accepted count gives i_t == -1, a read before this request's row
|
||||
# (and before the tensor for i_n == 0); a stale or too-large count
|
||||
# reads past the row. The state_idx <= 0 guard below rejects only
|
||||
# non-positive values, so an out-of-range read returning a garbage
|
||||
# positive int flowed into h0 + state_idx * stride_init_state_token
|
||||
# and faulted the SM.
|
||||
idx_in_row = (i_t >= 0) & (i_t < stride_indices_seq)
|
||||
if (i_t < 0) or (i_t >= stride_indices_seq):
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (i_n).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, (i_t).to(tl.int64) + _GUARD_BIAS)
|
||||
state_idx = tl.load(
|
||||
ssm_state_indices + i_n * stride_indices_seq + i_t,
|
||||
mask=idx_in_row,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
# Skip invalid state indices without exposing uninitialized output.
|
||||
# o is q.new_empty in the entry below, so the previous early return
|
||||
# left this request's rows holding whatever the allocator returned.
|
||||
if state_idx <= 0:
|
||||
zero = tl.zeros([BV], dtype=tl.float32).to(p_o.dtype.element_ty)
|
||||
for _ in range(0, T):
|
||||
tl.store(p_o, zero, mask=mask_v)
|
||||
p_o += HV * V
|
||||
return
|
||||
p_h0 = h0 + state_idx * stride_init_state_token
|
||||
else:
|
||||
@@ -255,6 +284,9 @@ def fused_sigmoid_gating_delta_rule_update(
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
guard_ctr=guard_telemetry.counter(
|
||||
guard_telemetry.SITE_GDN_DECODE_STATE, q.device
|
||||
),
|
||||
scale=scale,
|
||||
N=N,
|
||||
T=T,
|
||||
|
||||
@@ -14,6 +14,7 @@ import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker import guard_telemetry
|
||||
from vllm.utils.math_utils import RCP_LN2, cdiv, next_power_of_2
|
||||
|
||||
from .chunk_delta_h import chunk_gated_delta_rule_fwd_h
|
||||
@@ -81,6 +82,12 @@ def fused_recurrent_kda_fwd(
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
# Bounds-guard instrumentation: the same kernel object as fused_recurrent.py:249, so it
|
||||
# takes the same guard counter. Same site: the defect being guarded is
|
||||
# the kernel's, not the caller's.
|
||||
guard_ctr=guard_telemetry.counter(
|
||||
guard_telemetry.SITE_GDN_RECURRENT_STATE, q.device
|
||||
),
|
||||
scale=scale,
|
||||
N=N,
|
||||
T=T,
|
||||
|
||||
@@ -8,7 +8,7 @@ column, num_accepted_tokens is always 1 (the checkpoint and the compacted conv w
|
||||
already hold the accepted state), and the metadata carries what the post-sampling commit
|
||||
needs. Mirrors the Kimi-K3 KDA builder's RecoverSSM handling."""
|
||||
|
||||
from dataclasses import dataclass, field, fields
|
||||
from dataclasses import dataclass, field, fields, replace
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -23,6 +23,7 @@ from vllm.v1.attention.backends.gdn_attn import (
|
||||
from vllm.v1.attention.backends.recoverssm_metadata import (
|
||||
RecoverSSMMetadata,
|
||||
RecoverSSMPostprocessMetadata,
|
||||
clone_commit_tensors,
|
||||
recoverssm_request_indices,
|
||||
recoverssm_spec_rows,
|
||||
)
|
||||
@@ -74,22 +75,54 @@ class GDNRecoverSSMMetadata(GDNAttentionMetadata, RecoverSSMMetadata):
|
||||
block_size=c.block_size,
|
||||
)
|
||||
|
||||
def snapshot_for_deferred_commit(self) -> "GDNRecoverSSMMetadata":
|
||||
c = self.recoverssm_commit
|
||||
if c is None:
|
||||
return self
|
||||
return replace(
|
||||
self,
|
||||
recoverssm_commit=clone_commit_tensors(c, self.num_spec_decodes),
|
||||
)
|
||||
|
||||
|
||||
def _check_context_binding(context: Any, checkpoints: list[torch.Tensor]) -> None:
|
||||
"""A cached commit context must point at the layers' live KV cache.
|
||||
|
||||
The context stores raw base addresses of every layer's conv, state and
|
||||
replay tensors. One that outlived its cache (the profiling KV cache is
|
||||
torn down and re-allocated before serving) would keep that cache alive
|
||||
and, worse, commit into freed memory. Builders are re-created with the
|
||||
real cache, so a mismatch is a lifecycle bug: fail loudly."""
|
||||
bound = [t.data_ptr() for t in context.checkpoints]
|
||||
live = [t.data_ptr() for t in checkpoints]
|
||||
if bound != live:
|
||||
raise RuntimeError(
|
||||
"GDN RecoverSSM commit context is bound to a different KV cache than "
|
||||
"its layers (stale context after a KV cache re-allocation)"
|
||||
)
|
||||
|
||||
|
||||
class GDNRecoverSSMMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
# The verify/commit metadata is not staged for FULL graph replay yet; declaring
|
||||
# NEVER makes the runner fall back to PIECEWISE (attention compiled piecewise).
|
||||
_cudagraph_support = AttentionCGSupport.NEVER
|
||||
# Uniform spec-decode batches replay as FULL graphs: the base builder stages the
|
||||
# state indices, query offsets and accepted counts into persistent buffers and pads
|
||||
# the batch with null-state, zero-length rows, which the verify kernel skips. The
|
||||
# commit runs after sampling, outside the graph, on the staged tensors.
|
||||
_cudagraph_support = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
def __init__(self, kv_cache_spec, layer_names, vllm_config, device) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
# The builder may be created before the runner resolves the CUDA graph mode.
|
||||
self.use_full_cuda_graph = False
|
||||
max_reqs = vllm_config.scheduler_config.max_num_seqs
|
||||
self._ones = torch.ones(max_reqs, dtype=torch.int32, device=device)
|
||||
self._context: Any = None
|
||||
self._logged = False
|
||||
self._capturing = False
|
||||
|
||||
def _get_context(self):
|
||||
if self._context is not None:
|
||||
fc = self.vllm_config.compilation_config.static_forward_context
|
||||
_check_context_binding(
|
||||
self._context, [fc[name].kv_cache[1] for name in self.layer_names]
|
||||
)
|
||||
if self._context is None:
|
||||
from vllm.model_executor.layers.mamba.gdn.recoverssm_gdn import (
|
||||
GDNRecoverSSMCommitContext,
|
||||
@@ -120,6 +153,9 @@ class GDNRecoverSSMMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
m = common_attn_metadata
|
||||
rows = None
|
||||
if self.use_spec_decode and num_decode_draft_tokens_cpu is not None:
|
||||
if self._capturing and m.is_prefilling is None:
|
||||
# CUDA graph capture: dummy decode batches, every row a spec window.
|
||||
m = m.replace(is_prefilling=torch.zeros(m.num_reqs, dtype=torch.bool))
|
||||
if m.is_prefilling is None or m.is_prefilling.device.type != "cpu":
|
||||
raise ValueError("GDN RecoverSSM needs the CPU is_prefilling mask")
|
||||
# Positive counts only defeat the base builder's "no drafts -> regular
|
||||
@@ -144,7 +180,11 @@ class GDNRecoverSSMMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
n = meta.num_spec_decodes
|
||||
assert meta.spec_state_indices_tensor is not None
|
||||
base["spec_state_indices_tensor"] = meta.spec_state_indices_tensor[:, :1]
|
||||
base["num_accepted_tokens"] = self._ones[:n]
|
||||
# One accepted token per row: the checkpoint already holds the accepted
|
||||
# state. Sized like the (possibly graph-padded) state indices.
|
||||
base["num_accepted_tokens"] = self._ones[
|
||||
: base["spec_state_indices_tensor"].shape[0]
|
||||
]
|
||||
if rows is None:
|
||||
raise ValueError("GDN RecoverSSM: spec rows without draft counts")
|
||||
request_indices = recoverssm_request_indices(
|
||||
@@ -170,6 +210,15 @@ class GDNRecoverSSMMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
recoverssm_context=self._get_context() if commit is not None else None,
|
||||
)
|
||||
|
||||
def build_for_cudagraph_capture(
|
||||
self, common_attn_metadata: CommonAttentionMetadata
|
||||
):
|
||||
self._capturing = True
|
||||
try:
|
||||
return super().build_for_cudagraph_capture(common_attn_metadata)
|
||||
finally:
|
||||
self._capturing = False
|
||||
|
||||
|
||||
class GDNRecoverSSMAttentionBackend(GDNAttentionBackend):
|
||||
@staticmethod
|
||||
|
||||
@@ -9,7 +9,7 @@ every active decode row takes the spec path, the conv reads at offset 0, and aft
|
||||
sampling the accepted window is compacted to the front (the shared RecoverSSM compaction
|
||||
kernel; in align mode it also writes the block-boundary window)."""
|
||||
|
||||
from dataclasses import dataclass, field, fields
|
||||
from dataclasses import dataclass, field, fields, replace
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -25,6 +25,7 @@ from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.attention.backends.recoverssm_metadata import (
|
||||
RecoverSSMMetadata,
|
||||
RecoverSSMPostprocessMetadata,
|
||||
clone_commit_tensors,
|
||||
recoverssm_request_indices,
|
||||
recoverssm_spec_rows,
|
||||
)
|
||||
@@ -228,21 +229,43 @@ class PleRecoverSSMMetadata(PleShortConvAttentionMetadata, RecoverSSMMetadata):
|
||||
block_size=c.block_size,
|
||||
)
|
||||
|
||||
def snapshot_for_deferred_commit(self) -> "PleRecoverSSMMetadata":
|
||||
c = self.recoverssm_commit
|
||||
if c is None:
|
||||
return self
|
||||
return replace(
|
||||
self,
|
||||
recoverssm_commit=clone_commit_tensors(c, self.num_spec_decodes),
|
||||
)
|
||||
|
||||
|
||||
class PleRecoverSSMMetadataBuilder(PleShortConvAttentionMetadataBuilder):
|
||||
# Same as the GDN RecoverSSM builder: PIECEWISE only for now.
|
||||
_cudagraph_support = AttentionCGSupport.NEVER
|
||||
# Same as the GDN RecoverSSM builder: uniform spec-decode batches replay as FULL
|
||||
# graphs on the base builder's staged, null-padded buffers; the commit runs after
|
||||
# sampling, outside the graph.
|
||||
_cudagraph_support = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
def __init__(self, kv_cache_spec, layer_names, vllm_config, device) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
self.use_full_cuda_graph = False
|
||||
self._ones = torch.ones(
|
||||
vllm_config.scheduler_config.max_num_seqs, dtype=torch.int32, device=device
|
||||
)
|
||||
self._ctx: _PleConvCommit | None = None
|
||||
self._logged = False
|
||||
self._capturing = False
|
||||
|
||||
def _get_ctx(self):
|
||||
if self._ctx is not None:
|
||||
fc = self.vllm_config.compilation_config.static_forward_context
|
||||
live = [fc[name].kv_cache[0].data_ptr() for name in self.layer_names]
|
||||
if [s.data_ptr() for s in self._ctx.conv_states] != live:
|
||||
# Same lifecycle guard as the GDN builder: never commit into a
|
||||
# KV cache that was torn down (profiling) and re-allocated.
|
||||
raise RuntimeError(
|
||||
"PLE RecoverSSM commit context is bound to a different KV "
|
||||
"cache than its layers (stale context after a KV cache "
|
||||
"re-allocation)"
|
||||
)
|
||||
if self._ctx is None:
|
||||
fc = self.vllm_config.compilation_config.static_forward_context
|
||||
convs = [fc[name].kv_cache[0] for name in self.layer_names]
|
||||
@@ -270,6 +293,9 @@ class PleRecoverSSMMetadataBuilder(PleShortConvAttentionMetadataBuilder):
|
||||
m = common_attn_metadata
|
||||
rows = None
|
||||
if self.use_spec_decode and num_decode_draft_tokens_cpu is not None:
|
||||
if self._capturing and m.is_prefilling is None:
|
||||
# CUDA graph capture: dummy decode batches, every row a spec window.
|
||||
m = m.replace(is_prefilling=torch.zeros(m.num_reqs, dtype=torch.bool))
|
||||
if m.is_prefilling is None or m.is_prefilling.device.type != "cpu":
|
||||
raise ValueError("PLE RecoverSSM needs the CPU is_prefilling mask")
|
||||
# the same classification as the GDN builder, so both commits map the same
|
||||
@@ -295,7 +321,10 @@ class PleRecoverSSMMetadataBuilder(PleShortConvAttentionMetadataBuilder):
|
||||
return meta
|
||||
base = {f.name: getattr(meta, f.name) for f in fields(meta)}
|
||||
n = meta.num_spec_decodes
|
||||
base["num_accepted_tokens"] = self._ones[:n]
|
||||
# sized like the (possibly graph-padded) state indices
|
||||
assert meta.spec_state_indices_tensor is not None
|
||||
num_rows = meta.spec_state_indices_tensor.shape[0]
|
||||
base["num_accepted_tokens"] = self._ones[:num_rows]
|
||||
if rows is None:
|
||||
raise ValueError("PLE RecoverSSM: spec rows without draft counts")
|
||||
req_idx = recoverssm_request_indices(rows, n, m.query_start_loc.device)
|
||||
@@ -315,6 +344,13 @@ class PleRecoverSSMMetadataBuilder(PleShortConvAttentionMetadataBuilder):
|
||||
**base, recoverssm_commit=commit, recoverssm_context=self._get_ctx()
|
||||
)
|
||||
|
||||
def build_for_cudagraph_capture(self, common_attn_metadata):
|
||||
self._capturing = True
|
||||
try:
|
||||
return super().build_for_cudagraph_capture(common_attn_metadata)
|
||||
finally:
|
||||
self._capturing = False
|
||||
|
||||
|
||||
class PleRecoverSSMAttentionBackend(PleShortConvAttentionBackend):
|
||||
@staticmethod
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import abc
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import torch
|
||||
|
||||
@@ -25,6 +25,25 @@ class RecoverSSMMetadata(abc.ABC):
|
||||
) -> RecoverSSMPostprocessMetadata | None:
|
||||
raise NotImplementedError
|
||||
|
||||
def has_recoverssm_commit(self) -> bool:
|
||||
"""Whether this step's metadata carries a commit (any spec row)."""
|
||||
return getattr(self, "recoverssm_commit", None) is not None
|
||||
|
||||
def snapshot_for_deferred_commit(self) -> "RecoverSSMMetadata":
|
||||
"""A copy whose commit inputs no longer alias per-step buffers.
|
||||
|
||||
Under pipeline parallelism a non-last rank learns the accepted counts
|
||||
pp_size steps after its verify (the PP sampled-token broadcast), and
|
||||
the steps in between rebuild the builders' persistent, graph-staged
|
||||
buffers (spec state indices, query offsets, the batch-order block
|
||||
table). The deferred commit must read this step's values, so they are
|
||||
cloned here, on the current stream, before the next build overwrites
|
||||
them. Implementations that do not support the PP deferral raise."""
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} does not support the pipeline-parallel "
|
||||
"RecoverSSM commit deferral"
|
||||
)
|
||||
|
||||
|
||||
def recoverssm_spec_rows(
|
||||
is_prefilling_cpu: torch.Tensor,
|
||||
@@ -62,3 +81,23 @@ def recoverssm_request_indices(
|
||||
if num_spec_decodes == 0 or int(rows[-1]) == num_spec_decodes - 1:
|
||||
return None
|
||||
return rows.to(torch.int32).to(device, non_blocking=True)
|
||||
|
||||
|
||||
def clone_commit_tensors(commit, num_spec_decodes: int):
|
||||
"""Clone the commit inputs a deferred (PP) commit reads, trimmed to the real
|
||||
spec rows. Works for every RecoverSSM commit dataclass with the fields
|
||||
state_indices, query_start_loc, request_indices, block_table and
|
||||
num_computed_tokens."""
|
||||
n = num_spec_decodes
|
||||
|
||||
def c(t: torch.Tensor | None) -> torch.Tensor | None:
|
||||
return None if t is None else t.clone()
|
||||
|
||||
return replace(
|
||||
commit,
|
||||
state_indices=commit.state_indices[:n].clone(),
|
||||
query_start_loc=commit.query_start_loc[: n + 1].clone(),
|
||||
request_indices=c(commit.request_indices),
|
||||
block_table=c(commit.block_table),
|
||||
num_computed_tokens=c(commit.num_computed_tokens),
|
||||
)
|
||||
|
||||
@@ -264,8 +264,9 @@ def resolve_kv_cache_layout(
|
||||
"""Resolve one KV cache layout for the whole model.
|
||||
|
||||
Runs once in the engine core. Every worker reports the layouts its backends
|
||||
support, most preferred first (``get_supported_kv_cache_layouts``); all
|
||||
ranks run the same backends, so their lists must agree. Specs mixing HNC
|
||||
support, most preferred first (``get_supported_kv_cache_layouts``); ranks
|
||||
may expose different sets under pipeline parallelism, but their lists must
|
||||
share at least one layout. Specs mixing HNC
|
||||
shapes narrow the candidates to block-compact layouts. An explicit
|
||||
``VLLM_KV_CACHE_LAYOUT`` must be one of the candidates or resolution fails,
|
||||
with the legacy ``NHD``/``HND`` names as aliases for ``LBNHC``/``LBHNC``; the
|
||||
@@ -282,10 +283,20 @@ def resolve_kv_cache_layout(
|
||||
assert supported_layouts and all(supported_layouts), (
|
||||
"No worker reported supported KV cache layouts."
|
||||
)
|
||||
assert all(names == supported_layouts[0] for names in supported_layouts[1:]), (
|
||||
f"Workers disagree on supported KV cache layouts: {supported_layouts}."
|
||||
)
|
||||
candidates = [_layout_from_name(name) for name in supported_layouts[0]]
|
||||
# Fork: restore PP fix #54795 from the E2 base, dropped
|
||||
# in the v0.30.0 rebase. Under pipeline parallelism the stages run
|
||||
# different backends (PP0 carries the PLE short-conv, PP1 the drafter), so
|
||||
# their lists need only share a layout, not be identical.
|
||||
common_layout_names = [
|
||||
name
|
||||
for name in supported_layouts[0]
|
||||
if all(name in names for names in supported_layouts[1:])
|
||||
]
|
||||
if not common_layout_names:
|
||||
raise ValueError(
|
||||
f"Workers share no supported KV cache layout: {supported_layouts}."
|
||||
)
|
||||
candidates = [_layout_from_name(name) for name in common_layout_names]
|
||||
|
||||
# A block-compact layout means the block is densely packed in memory, so any mix of
|
||||
# specs can re-interpret HNC with different sizes as long as the total number of
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
KVCacheBlock,
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
CircularBufferManager,
|
||||
CrossAttentionManager,
|
||||
MambaManager,
|
||||
SingleTypeKVCacheManager,
|
||||
@@ -147,6 +148,17 @@ class KVCacheCoordinator(ABC):
|
||||
)
|
||||
for i, kv_cache_group in enumerate(self.kv_cache_config.kv_cache_groups)
|
||||
)
|
||||
# v2.5 side-cache layout: a slot-packed ring group shares each pool
|
||||
# block between several requests (kv_cache_utils._choose_ring_slots).
|
||||
for kv_cache_group, manager in zip(
|
||||
self.kv_cache_config.kv_cache_groups, self.single_type_managers
|
||||
):
|
||||
if kv_cache_group.slots_per_block > 1:
|
||||
assert type(manager) is CircularBufferManager, (
|
||||
f"slot packing is only defined for the QSA ring, got "
|
||||
f"{type(manager).__name__}"
|
||||
)
|
||||
manager.enable_slot_packing(kv_cache_group.slots_per_block)
|
||||
# Match Mamba checkpoints to Eagle's attention replay boundary.
|
||||
if use_eagle:
|
||||
for manager in self.single_type_managers:
|
||||
@@ -233,6 +245,20 @@ class KVCacheCoordinator(ABC):
|
||||
apply_admission_cap=apply_admission_cap,
|
||||
prefill_end=prefill_end,
|
||||
)
|
||||
if apply_admission_cap:
|
||||
# Full-sequence admission: a retained Mamba checkpoint is
|
||||
# held outside the block table for the request's
|
||||
# lifetime, so reserve it here (v2.5). Per-step
|
||||
# allocation needs no term: a held block is simply not in
|
||||
# the free queue that check reads.
|
||||
num_blocks_to_allocate += (
|
||||
manager.get_num_retained_checkpoint_blocks_to_reserve(
|
||||
request_id,
|
||||
num_tokens_main_model,
|
||||
new_computed_blocks[i],
|
||||
num_local_computed_tokens,
|
||||
)
|
||||
)
|
||||
return num_blocks_to_allocate
|
||||
|
||||
def allocate_new_computed_blocks(
|
||||
@@ -395,26 +421,42 @@ class KVCacheCoordinator(ABC):
|
||||
request_id: The request ID.
|
||||
|
||||
"""
|
||||
# Retained Mamba checkpoints go back last, after every group's table,
|
||||
# so a finished request's checkpoint is never evicted ahead of the
|
||||
# attention prefix it resumes (v2.5).
|
||||
retained: list[KVCacheBlock] = []
|
||||
for manager in self.single_type_managers:
|
||||
retained.extend(manager.pop_retained_checkpoints(request_id))
|
||||
manager.free(request_id)
|
||||
if retained:
|
||||
self.block_pool.free_blocks(retained)
|
||||
|
||||
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
|
||||
"""Pop the request's bookkeeping from all single-type managers and
|
||||
return its blocks without returning them to the block pool. The
|
||||
caller must eventually pass the returned blocks to
|
||||
`block_pool.free_blocks`, freeing them in reverse order (so that
|
||||
tail blocks are evicted first).
|
||||
`block_pool.free_blocks` as-is: they are already ordered so that
|
||||
tail blocks are evicted first, within each KV cache group.
|
||||
|
||||
Args:
|
||||
request_id: The request ID.
|
||||
|
||||
Returns:
|
||||
The request's blocks in allocation order.
|
||||
The request's blocks in eviction order (each group's blocks
|
||||
reversed, then concatenated group by group).
|
||||
|
||||
"""
|
||||
blocks: list[KVCacheBlock] = []
|
||||
retained: list[KVCacheBlock] = []
|
||||
for manager in self.single_type_managers:
|
||||
blocks.extend(manager.pop_blocks_for_free(request_id))
|
||||
retained.extend(manager.pop_retained_checkpoints(request_id))
|
||||
# Reverse within each group so the concatenation is already in
|
||||
# eviction order, matching `free()`, which frees each group in its
|
||||
# own reversed `free_blocks()` call. Reversing the flat list
|
||||
# instead would emit whole groups in the wrong order.
|
||||
blocks.extend(reversed(manager.pop_blocks_for_free(request_id)))
|
||||
# Retained checkpoints last, as in `free()`.
|
||||
blocks.extend(retained)
|
||||
return blocks
|
||||
|
||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> list[int]:
|
||||
|
||||
@@ -223,6 +223,52 @@ class KVCacheManager:
|
||||
tuple(() for _ in range(self.num_kv_cache_groups))
|
||||
)
|
||||
|
||||
# Prefix-cache hits pinned for requests waiting for admission
|
||||
# ({request_id: blocks}). A waiting follow-up turn holds no ref on its
|
||||
# cached prefix, and with a prefix unit finer than the block its
|
||||
# whole hit depends on every one of those blocks (the resume key sits
|
||||
# in the top block); under pool pressure other requests' allocations
|
||||
# popped them from the free queue and the turn re-prefilled from 0
|
||||
# (under a long-context multi-turn agent load: 4-5 total misses per
|
||||
# 300 s at unit 64/512, none at 4096). Pinned blocks stay out of the
|
||||
# free queue, so they count against admission and the watermark like
|
||||
# any referenced block; they
|
||||
# are released on admission (the table takes over), on free, and
|
||||
# whenever a running request would otherwise fail to allocate
|
||||
# (``yield_pinned_hits``), so a pin never causes a preemption.
|
||||
self._pinned_hits: dict[str, list[KVCacheBlock]] = {}
|
||||
# A finished request's hashed blocks (its attention prefix and
|
||||
# the GDN states it holds) stay referenced as a "grace" pin until the
|
||||
# conversation's next turn is admitted on them. With waiter pins alone,
|
||||
# every remaining total miss had lost its top attention block BEFORE
|
||||
# the next turn was first looked up -- in the gap between the
|
||||
# previous turn's free and the next turn's arrival, or while its
|
||||
# admission kept failing -- when nothing referenced the prefix and
|
||||
# the running set's allocations popped it; once the hybrid hit is 0
|
||||
# a waiter pin has nothing to pin. Grace pins yield first (oldest
|
||||
# first) whenever anyone needs blocks, so they never cost a running
|
||||
# request or an admission anything a free queue would have given.
|
||||
# A grace is one extra ref per block, taken on whichever free path
|
||||
# the request takes (immediate or deferred behind an in-flight step).
|
||||
self._grace_pins: dict[str, list[KVCacheBlock]] = {}
|
||||
self._grace_block_ids: dict[str, frozenset[int]] = {}
|
||||
self._grace_prompt_tokens: dict[str, int] = {}
|
||||
# Only for fine-grained hybrid hits (align Mamba hashed finer than its
|
||||
# block): there the next turn's key sits in the top attention block
|
||||
# and one lost block is a total miss; at a block-sized unit the top
|
||||
# block is a spare and nothing changes. The scheduler also turns
|
||||
# graces off when a KV connector is configured (as for waiter pins).
|
||||
self.enable_grace_pins = (
|
||||
self.enable_caching and self.coordinator.enable_partial_hash_hits
|
||||
)
|
||||
# Shortfall (blocks) of the last allocate_slots that returned None.
|
||||
self.last_alloc_shortfall = 0
|
||||
# A producer that moves its live GDN tail's hash to a CoW copy keeps
|
||||
# writing the source; pins and graces follow the hash to the copy.
|
||||
for manager in self.coordinator.single_type_managers:
|
||||
if isinstance(manager, MambaManager):
|
||||
manager.moved_hash_listeners.append(self._pins_follow_moved_hash)
|
||||
|
||||
@property
|
||||
def usage(self) -> float:
|
||||
"""Get the KV cache usage.
|
||||
@@ -536,6 +582,9 @@ class KVCacheManager:
|
||||
)
|
||||
required_blocks = num_blocks_to_allocate + watermark_blocks
|
||||
if required_blocks > self.block_pool.get_num_free_blocks():
|
||||
self.last_alloc_shortfall = (
|
||||
required_blocks - self.block_pool.get_num_free_blocks()
|
||||
)
|
||||
return None
|
||||
|
||||
num_tokens_main_model = total_computed_tokens + num_new_tokens
|
||||
@@ -576,7 +625,9 @@ class KVCacheManager:
|
||||
required_blocks = num_blocks_to_allocate + watermark_blocks
|
||||
if required_blocks > available_blocks:
|
||||
# Cannot allocate new blocks
|
||||
self.last_alloc_shortfall = required_blocks - available_blocks
|
||||
return None
|
||||
self.last_alloc_shortfall = 0
|
||||
|
||||
if (
|
||||
new_computed_block_list is not self.empty_kv_cache_blocks.blocks
|
||||
@@ -602,6 +653,11 @@ class KVCacheManager:
|
||||
# P/D: delay caching blocks if we have to recv from
|
||||
# remote. Update state for locally cached blocks.
|
||||
if delay_cache_blocks:
|
||||
self.release_pinned_hit(request.request_id)
|
||||
if self._grace_pins and num_new_computed_tokens > 0:
|
||||
self._release_grace_for_hit(
|
||||
new_computed_block_list, num_local_computed_tokens
|
||||
)
|
||||
return self.create_kv_cache_blocks(new_blocks)
|
||||
|
||||
# NOTE(woosuk): We want to commit (cache) up to num_local_computed_tokens
|
||||
@@ -615,6 +671,13 @@ class KVCacheManager:
|
||||
)
|
||||
self.coordinator.cache_blocks(request, num_tokens_to_cache)
|
||||
|
||||
# Admitted: the table now references the hit, the pins are redundant.
|
||||
self.release_pinned_hit(request.request_id)
|
||||
if self._grace_pins and num_new_computed_tokens > 0:
|
||||
# ... and so does the grace pin of the turn it resumes.
|
||||
self._release_grace_for_hit(
|
||||
new_computed_block_list, num_local_computed_tokens
|
||||
)
|
||||
return self.create_kv_cache_blocks(new_blocks)
|
||||
|
||||
def free(self, request: Request) -> None:
|
||||
@@ -626,7 +689,331 @@ class KVCacheManager:
|
||||
request: The request to free the blocks.
|
||||
|
||||
"""
|
||||
self.coordinator.free(request.request_id)
|
||||
self.release_pinned_hit(request.request_id)
|
||||
if not self._grace_eligible(request):
|
||||
self.coordinator.free(request.request_id)
|
||||
return
|
||||
# Same blocks and order as coordinator.free (the deferred path frees
|
||||
# this list as-is too); the grace takes its ref before the table's
|
||||
# goes, so the hashed blocks never pass through the free queue.
|
||||
blocks = self.coordinator.pop_blocks_for_free(request.request_id)
|
||||
self._take_grace_pin(request, blocks)
|
||||
self.block_pool.free_blocks(blocks)
|
||||
|
||||
# Finished requests whose hashed blocks stay pinned for the next turn
|
||||
# (oldest released first beyond this). One per conversation is the
|
||||
# need; twice the deployed max_num_seqs.
|
||||
MAX_GRACE_PINS = 16
|
||||
|
||||
@property
|
||||
def num_grace_pins(self) -> int:
|
||||
return len(self._grace_pins)
|
||||
|
||||
@property
|
||||
def num_grace_pinned_blocks(self) -> int:
|
||||
return sum(len(blocks) for blocks in self._grace_pins.values())
|
||||
|
||||
def grace_pinned_blocks(self, request_id: str) -> list[KVCacheBlock]:
|
||||
return list(self._grace_pins.get(request_id, ()))
|
||||
|
||||
def _grace_eligible(self, request: Request) -> bool:
|
||||
"""A turn that finished with output (stop, length, repetition, or
|
||||
an abort after decoding) may have a next turn that resumes it. A
|
||||
preemption (still RUNNING), an aborted waiter (no output), an error
|
||||
or an ignored request leaves nothing worth holding."""
|
||||
return (
|
||||
self.enable_grace_pins
|
||||
and self.enable_caching
|
||||
and self.prefix_cache_lookup_enabled(request)
|
||||
and request.is_finished()
|
||||
and request.status
|
||||
not in (RequestStatus.FINISHED_ERROR, RequestStatus.FINISHED_IGNORED)
|
||||
and request.num_output_tokens > 0
|
||||
)
|
||||
|
||||
def _take_grace_pin(self, request: Request, blocks: list[KVCacheBlock]) -> int:
|
||||
"""Take one extra ref on every hashed block of a finished request's
|
||||
just-popped blocks (``blocks`` is in eviction order: each group's
|
||||
table tail first, the held GDN states last). The caller then drops
|
||||
the table refs, now or after the in-flight fence; the hashed blocks
|
||||
stay out of the free queue until the grace is released. Unhashed
|
||||
blocks (running state, MTP scratch, an unkeyed last block) go back
|
||||
with the table refs as before. Returns the number of blocks held."""
|
||||
keep: list[KVCacheBlock] = []
|
||||
seen: set[int] = set()
|
||||
for block in blocks:
|
||||
if block.is_null or block.block_hash is None or id(block) in seen:
|
||||
continue
|
||||
# Still referenced by the request's table or hold: the grace ref
|
||||
# never moves a block in or out of a free queue (any pool).
|
||||
assert block.ref_cnt > 0
|
||||
seen.add(id(block))
|
||||
keep.append(block)
|
||||
if not keep:
|
||||
return 0
|
||||
request_id = request.request_id
|
||||
self.release_grace_pin(request_id) # a repeated free: replace
|
||||
for block in keep:
|
||||
block.ref_cnt += 1
|
||||
self._grace_pins[request_id] = keep
|
||||
self._grace_block_ids[request_id] = frozenset(seen)
|
||||
self._grace_prompt_tokens[request_id] = request.num_prompt_tokens
|
||||
while len(self._grace_pins) > self.MAX_GRACE_PINS:
|
||||
self.release_grace_pin(next(iter(self._grace_pins)))
|
||||
return len(keep)
|
||||
|
||||
def release_grace_pin(self, request_id: str) -> int:
|
||||
"""Return a finished request's grace-pinned blocks to the pool (in
|
||||
the eviction order a free would have used). Returns the count."""
|
||||
blocks = self._grace_pins.pop(request_id, None)
|
||||
self._grace_block_ids.pop(request_id, None)
|
||||
self._grace_prompt_tokens.pop(request_id, None)
|
||||
if not blocks:
|
||||
return 0
|
||||
self.block_pool.free_blocks(blocks)
|
||||
return len(blocks)
|
||||
|
||||
def _release_grace_for_hit(
|
||||
self,
|
||||
computed_blocks: tuple[Sequence[KVCacheBlock], ...],
|
||||
num_hit_tokens: int,
|
||||
) -> None:
|
||||
"""An admitted hit now references its blocks through the table, so
|
||||
the grace of the turn it resumes has done its job. That turn is a
|
||||
grace that holds the deepest hit block of every group whose deepest
|
||||
hit block any grace holds (the GDN state it resumes from, the
|
||||
attention block holding the key), and whose prompt the hit reaches
|
||||
to within two prefix units (the EAGLE-dropped tail key). A new turn
|
||||
that only shares a system prompt with finished conversations
|
||||
releases none of theirs."""
|
||||
candidates: set[str] | None = None
|
||||
for group in computed_blocks:
|
||||
deepest = next((b for b in reversed(group) if not b.is_null), None)
|
||||
if deepest is None:
|
||||
continue
|
||||
owners = {
|
||||
request_id
|
||||
for request_id, block_ids in self._grace_block_ids.items()
|
||||
if id(deepest) in block_ids
|
||||
}
|
||||
if owners:
|
||||
candidates = owners if candidates is None else candidates & owners
|
||||
if not candidates:
|
||||
return
|
||||
reach = num_hit_tokens + 2 * self.block_pool.hash_block_size
|
||||
for request_id in candidates:
|
||||
if reach >= self._grace_prompt_tokens.get(request_id, 0):
|
||||
self.release_grace_pin(request_id)
|
||||
|
||||
def _pins_follow_moved_hash(
|
||||
self, source_block: KVCacheBlock, cow_block: KVCacheBlock
|
||||
) -> None:
|
||||
"""A Mamba producer moved ``source_block``'s hashes to ``cow_block``
|
||||
and keeps writing ``source_block``. A grace or waiter pin on the
|
||||
source follows to the copy (as a hold does), so it protects the
|
||||
hashed state and never leaves an extra ref on a block its owner must
|
||||
hold exclusively (speculative relocation asserts on that)."""
|
||||
for request_id, blocks in self._grace_pins.items():
|
||||
if id(source_block) in self._grace_block_ids[request_id]:
|
||||
self._swap_pinned_block(blocks, source_block, cow_block)
|
||||
self._grace_block_ids[request_id] = frozenset(
|
||||
id(block) for block in blocks
|
||||
)
|
||||
for blocks in self._pinned_hits.values():
|
||||
self._swap_pinned_block(blocks, source_block, cow_block)
|
||||
|
||||
def _swap_pinned_block(
|
||||
self,
|
||||
blocks: list[KVCacheBlock],
|
||||
source_block: KVCacheBlock,
|
||||
cow_block: KVCacheBlock,
|
||||
) -> None:
|
||||
for i, block in enumerate(blocks):
|
||||
if block is source_block:
|
||||
if any(other is cow_block for other in blocks):
|
||||
del blocks[i]
|
||||
else:
|
||||
assert cow_block.ref_cnt > 0
|
||||
cow_block.ref_cnt += 1
|
||||
blocks[i] = cow_block
|
||||
self.block_pool.free_blocks([source_block])
|
||||
return
|
||||
|
||||
def num_yieldable_blocks(
|
||||
self,
|
||||
keep: str | None = None,
|
||||
keep_hit: tuple[Sequence[KVCacheBlock], ...] | None = None,
|
||||
) -> int:
|
||||
"""Blocks that would return to the free queue if every grace and
|
||||
every waiter pin except ``keep``'s were released now. With
|
||||
``keep_hit`` (the hit ``keep`` is being admitted on), ``keep``'s
|
||||
pinned blocks outside that hit (the proof block past it) count too:
|
||||
``release_pinned_extras`` can give them back. Blocks of that hit
|
||||
never count: freed, they are taken straight back by the admission
|
||||
(a hit that grew past the pin snapshot would otherwise be counted
|
||||
through the grace that still holds it)."""
|
||||
pins: dict[int, list] = {}
|
||||
holders = [*self._grace_pins.values()] + [
|
||||
blocks
|
||||
for request_id, blocks in self._pinned_hits.items()
|
||||
if request_id != keep
|
||||
]
|
||||
hit_ids: set[int] = set()
|
||||
if keep is not None and keep_hit is not None:
|
||||
holders.append(self._pinned_extras(keep, keep_hit))
|
||||
hit_ids = {id(block) for group in keep_hit for block in group}
|
||||
for blocks in holders:
|
||||
for block in blocks:
|
||||
entry = pins.setdefault(id(block), [block, 0])
|
||||
entry[1] += 1
|
||||
return sum(
|
||||
1
|
||||
for block_id, (block, n) in pins.items()
|
||||
if block.ref_cnt == n and block_id not in hit_ids
|
||||
)
|
||||
|
||||
def _pinned_extras(
|
||||
self, request_id: str, hit: tuple[Sequence[KVCacheBlock], ...]
|
||||
) -> list[KVCacheBlock]:
|
||||
hit_ids = {id(block) for group in hit for block in group}
|
||||
return [
|
||||
block
|
||||
for block in self._pinned_hits.get(request_id, ())
|
||||
if id(block) not in hit_ids
|
||||
]
|
||||
|
||||
def release_pinned_extras(
|
||||
self, request_id: str, hit: tuple[Sequence[KVCacheBlock], ...]
|
||||
) -> int:
|
||||
"""Drop from the request's pin the blocks outside ``hit``: the proof
|
||||
block past it, and any hit block a later lookup added. The request
|
||||
is being admitted on ``hit`` now, so they protect nothing it still
|
||||
needs; they cost it blocks. Returns the number dropped."""
|
||||
extras = self._pinned_extras(request_id, hit)
|
||||
if not extras:
|
||||
return 0
|
||||
extra_ids = {id(block) for block in extras}
|
||||
rest = [
|
||||
block
|
||||
for block in self._pinned_hits[request_id]
|
||||
if id(block) not in extra_ids
|
||||
]
|
||||
if rest:
|
||||
self._pinned_hits[request_id] = rest
|
||||
else:
|
||||
del self._pinned_hits[request_id]
|
||||
self.block_pool.free_blocks(extras)
|
||||
return len(extras)
|
||||
|
||||
# Waiting requests whose prefix-cache hit is pinned at once (arrival
|
||||
# order). Pins never block anyone (they yield before a running request
|
||||
# fails to allocate), so the bound only caps the lookup work per step;
|
||||
# twice the deployed max_num_seqs covers every waiter seen under load.
|
||||
MAX_PINNED_WAITERS = 16
|
||||
|
||||
@property
|
||||
def num_pinned_waiters(self) -> int:
|
||||
return len(self._pinned_hits)
|
||||
|
||||
@property
|
||||
def num_pinned_blocks(self) -> int:
|
||||
return sum(len(blocks) for blocks in self._pinned_hits.values())
|
||||
|
||||
def pin_waiting_hit(self, request: Request) -> int:
|
||||
"""Pin the request's current prefix-cache hit while it waits for
|
||||
admission: every non-null hit block in every group (the attention
|
||||
prefix up to the key and the GDN state it resumes from) takes a ref
|
||||
and leaves the free queue. Idempotent. Returns the number of blocks
|
||||
pinned now (0 if nothing hit, already pinned, caching is off, or the
|
||||
waiter bound is reached).
|
||||
|
||||
Known gap: a pin is a snapshot of the hit at pin time. It is not
|
||||
refreshed if the hit grows while the request waits (a sibling
|
||||
computing more of the same prefix); the admission lookup still finds
|
||||
whatever is cached then, so the only cost is a smaller hit, never a
|
||||
wrong one. A producer moving the hash of a pinned live GDN block into
|
||||
its CoW copy is followed (``_pins_follow_moved_hash``)."""
|
||||
request_id = request.request_id
|
||||
if (
|
||||
request_id in self._pinned_hits
|
||||
or not self.enable_caching
|
||||
or not self.prefix_cache_lookup_enabled(request)
|
||||
or len(self._pinned_hits) >= self.MAX_PINNED_WAITERS
|
||||
):
|
||||
return 0
|
||||
computed, num_hit, _ = self.get_computed_blocks(request)
|
||||
blocks = [
|
||||
block
|
||||
for group in computed.blocks
|
||||
for block in group
|
||||
if not block.is_null
|
||||
]
|
||||
if num_hit == 0 or not blocks:
|
||||
return 0
|
||||
# The EAGLE/MTP proof: fine-grained attention hits match one hash
|
||||
# unit past the key and drop it, so the hit also depends on the block
|
||||
# holding that unit -- a different block when the key ends a block
|
||||
# (and never part of the returned hit). Pin it too.
|
||||
unit = self.block_pool.hash_block_size
|
||||
proof_idx = num_hit // unit
|
||||
if proof_idx < len(request.block_hashes):
|
||||
seen = {block.block_id for block in blocks}
|
||||
for group_id in range(self.num_kv_cache_groups):
|
||||
cached = self.block_pool.get_cached_block(
|
||||
request.block_hashes[proof_idx], [group_id]
|
||||
)
|
||||
if cached and cached[0].block_id not in seen:
|
||||
seen.add(cached[0].block_id)
|
||||
blocks.append(cached[0])
|
||||
# Blocks of a private-pool group (hisparse host tier) are touched in
|
||||
# their own pool, as free_blocks routes them.
|
||||
for block in blocks:
|
||||
(block.pool or self.block_pool).touch([block])
|
||||
self._pinned_hits[request_id] = blocks
|
||||
return len(blocks)
|
||||
|
||||
def pinned_hit_blocks(self, request_id: str) -> list[KVCacheBlock]:
|
||||
"""The blocks pinned for ``request_id`` (empty if none)."""
|
||||
return list(self._pinned_hits.get(request_id, ()))
|
||||
|
||||
def release_pinned_hit(self, request_id: str) -> int:
|
||||
"""Drop the request's pin (admission, abort, finish). Returns the
|
||||
number of blocks released."""
|
||||
blocks = self._pinned_hits.pop(request_id, None)
|
||||
if not blocks:
|
||||
return 0
|
||||
# Reversed: the prefix stays an eviction candidate longer than the
|
||||
# tail, as after a free.
|
||||
self.block_pool.free_blocks(reversed(blocks))
|
||||
return len(blocks)
|
||||
|
||||
def yield_pinned_hits(
|
||||
self, keep: str | None = None, limit: int | None = None
|
||||
) -> int:
|
||||
"""Drop pins (except ``keep``'s) so a running request can allocate,
|
||||
or a waiter ahead in line can be admitted: grace pins of finished
|
||||
turns first (oldest first), then the most recently pinned waiters, at
|
||||
most ``limit`` pins per call (all by default; the scheduler passes 1
|
||||
and retries, so a decode step that needs a block costs one pin, not
|
||||
every pin). Returns the number of blocks released; 0 means there was
|
||||
nothing to yield."""
|
||||
released = 0
|
||||
yielded = 0
|
||||
# Grace pins first (oldest finished conversation first), then the
|
||||
# waiters' pins (most recently pinned first).
|
||||
for request_id in list(self._grace_pins):
|
||||
if limit is not None and yielded >= limit:
|
||||
return released
|
||||
released += self.release_grace_pin(request_id)
|
||||
yielded += 1
|
||||
for request_id in reversed(list(self._pinned_hits)):
|
||||
if request_id == keep:
|
||||
continue
|
||||
if limit is not None and yielded >= limit:
|
||||
break
|
||||
released += self.release_pinned_hit(request_id)
|
||||
yielded += 1
|
||||
return released
|
||||
|
||||
def remove_skipped_blocks(
|
||||
self,
|
||||
@@ -651,16 +1038,26 @@ class KVCacheManager:
|
||||
def pop_blocks_for_free(self, request: Request) -> list[KVCacheBlock]:
|
||||
"""Pop the request's bookkeeping and return its blocks without
|
||||
returning them to the block pool. The caller must eventually free
|
||||
them in reverse order (so that tail blocks are evicted first).
|
||||
them as-is: they are already ordered so that tail blocks are
|
||||
evicted first, within each KV cache group.
|
||||
|
||||
Args:
|
||||
request: The request to pop the blocks for.
|
||||
|
||||
Returns:
|
||||
The request's blocks in allocation order.
|
||||
The request's blocks in eviction order (each group's blocks
|
||||
reversed, then concatenated group by group).
|
||||
|
||||
"""
|
||||
return self.coordinator.pop_blocks_for_free(request.request_id)
|
||||
# A pin is never written by an in-flight step (the waiter has no
|
||||
# table), so it is released now, not with the deferred blocks.
|
||||
self.release_pinned_hit(request.request_id)
|
||||
blocks = self.coordinator.pop_blocks_for_free(request.request_id)
|
||||
# A finished turn whose last step is still in flight: the grace takes
|
||||
# its ref now; the table refs go when the fence drains.
|
||||
if self._grace_eligible(request):
|
||||
self._take_grace_pin(request, blocks)
|
||||
return blocks
|
||||
|
||||
def evict_blocks(self, block_ids: set[int]) -> None:
|
||||
"""Evict blocks from the prefix cache by their block IDs.
|
||||
@@ -681,6 +1078,8 @@ class KVCacheManager:
|
||||
False otherwise.
|
||||
|
||||
"""
|
||||
# Pinned waiters' hits are about to be invalidated anyway.
|
||||
self.yield_pinned_hits()
|
||||
if not self.coordinator.reset_prefix_cache():
|
||||
return False
|
||||
if self.log_stats:
|
||||
@@ -721,6 +1120,12 @@ class KVCacheManager:
|
||||
group.
|
||||
|
||||
"""
|
||||
if self._pinned_hits or self._grace_pins:
|
||||
# A pinned or graced block carries a ref that is not a running
|
||||
# request's, so the ref count can equal the running count for a
|
||||
# block only some of them share. Report no common prefix (cascade
|
||||
# attention is an optimisation; a wrong prefix would be read).
|
||||
return [0] * len(self.kv_cache_config.kv_cache_groups)
|
||||
return self.coordinator.get_num_common_prefix_blocks(running_request_id)
|
||||
|
||||
def take_events(self) -> list[KVCacheEvent]:
|
||||
@@ -867,6 +1272,21 @@ class KVCacheManager:
|
||||
ids.extend(mgr.take_new_block_ids())
|
||||
return ids
|
||||
|
||||
def take_new_ring_block_ids(self) -> list[int]:
|
||||
"""Drain ring blocks claimed since the last call (ring-row tag clear).
|
||||
Independent of KV zeroing: every claim is returned."""
|
||||
ids: list[int] = []
|
||||
for mgr in self.coordinator.single_type_managers:
|
||||
ids.extend(mgr.take_new_ring_block_ids())
|
||||
return ids
|
||||
|
||||
def get_ring_block_ids(self, request_id: str) -> list[int]:
|
||||
"""The request's ring (CircularBufferSpec) block ids, all groups."""
|
||||
ids: list[int] = []
|
||||
for mgr in self.coordinator.single_type_managers:
|
||||
ids.extend(mgr.get_ring_block_ids(request_id))
|
||||
return ids
|
||||
|
||||
def record_blocks_for_zeroing(self, request_id: str, start_token: int) -> None:
|
||||
"""Re-record the request's blocks from start_token onwards for
|
||||
zeroing, e.g. blocks a failed async KV load left unwritten.
|
||||
|
||||
@@ -1121,7 +1121,19 @@ def get_max_concurrency_for_kv_cache_config(
|
||||
"""
|
||||
num_blocks_per_request = 0
|
||||
host_blocks_per_request = 0
|
||||
# v2.5 side-cache layout: a slot-packed ring group shares each pool block
|
||||
# between `slots_per_block` requests, so it costs cdiv(max_num_seqs, slots)
|
||||
# blocks for the whole pool instead of one block per request.
|
||||
shared_blocks = 0
|
||||
max_num_seqs = getattr(
|
||||
getattr(vllm_config, "scheduler_config", None), "max_num_seqs", 1
|
||||
)
|
||||
if not isinstance(max_num_seqs, int) or max_num_seqs < 1:
|
||||
max_num_seqs = 1
|
||||
for group in kv_cache_config.kv_cache_groups:
|
||||
if group.slots_per_block > 1 and not group.host_resident:
|
||||
shared_blocks += cdiv(max_num_seqs, group.slots_per_block)
|
||||
continue
|
||||
required = cdiv(
|
||||
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
|
||||
group.kv_cache_spec.page_size_bytes,
|
||||
@@ -1130,7 +1142,9 @@ def get_max_concurrency_for_kv_cache_config(
|
||||
host_blocks_per_request += required
|
||||
else:
|
||||
num_blocks_per_request += required
|
||||
limits = [kv_cache_config.num_blocks / num_blocks_per_request]
|
||||
limits = [
|
||||
max(kv_cache_config.num_blocks - shared_blocks, 0) / num_blocks_per_request
|
||||
]
|
||||
if host_blocks_per_request:
|
||||
assert kv_cache_config.hisparse_host_num_blocks is not None
|
||||
limits.append(
|
||||
@@ -1806,18 +1820,47 @@ def get_kv_cache_config_from_groups(
|
||||
layers_by_spec: defaultdict[KVCacheSpec, list[str]] = defaultdict(list)
|
||||
if isinstance(group_spec, UniformTypeKVCacheSpecs):
|
||||
for layer_name, spec in group_spec.kv_cache_specs.items():
|
||||
layers_by_spec[spec].append(layer_name)
|
||||
# Fork (E2 PP fix #54793, dropped in the v0.30.0 rebase, restored
|
||||
# the vLLM 0.30 rebase): _project_kv_cache_groups_to_worker keeps the GLOBAL
|
||||
# UniformTypeKVCacheSpecs for a group with no layer on this PP
|
||||
# stage, so without this filter the worker gets a tensor for
|
||||
# another stage's layer and allocate_kv_cache's group lookup
|
||||
# raises StopIteration (Flash-Next PP1: the layer-2 PLE state).
|
||||
if layer_name in group.layer_names:
|
||||
layers_by_spec[spec].append(layer_name)
|
||||
elif group.layer_names:
|
||||
layers_by_spec[group_spec].extend(group.layer_names)
|
||||
|
||||
# v2.5 side-cache layout: a slot-packed group views every pool block as
|
||||
# `slots` equal slots, so its block stride is one slot and its view has
|
||||
# num_blocks * slots blocks. Pages of its layers must fit in one slot.
|
||||
slots = group.slots_per_block
|
||||
group_block_stride = interleaved_block_stride
|
||||
group_num_blocks = num_blocks
|
||||
if slots > 1:
|
||||
assert layout.is_block_outermost, (
|
||||
"Slot-packed KV cache groups need a block-outermost layout"
|
||||
)
|
||||
assert bytes_per_block % slots == 0, (bytes_per_block, slots)
|
||||
group_block_stride = bytes_per_block // slots
|
||||
group_num_blocks = num_blocks * slots
|
||||
group_bytes = sum(
|
||||
spec.page_size_bytes * len(names)
|
||||
for spec, names in layers_by_spec.items()
|
||||
)
|
||||
assert group_bytes <= group_block_stride, (
|
||||
f"slot-packed group needs {group_bytes} B per slot, slot is "
|
||||
f"{group_block_stride} B"
|
||||
)
|
||||
|
||||
byte_offset = 0
|
||||
for spec, layer_names in layers_by_spec.items():
|
||||
layer_stride, block_stride, _, _, _ = compute_layout_strides(
|
||||
spec,
|
||||
num_blocks,
|
||||
group_num_blocks,
|
||||
len(layer_names),
|
||||
layout,
|
||||
fixed_strides=(None, interleaved_block_stride, None, None, None),
|
||||
fixed_strides=(None, group_block_stride, None, None, None),
|
||||
)
|
||||
offset = (
|
||||
byte_offset
|
||||
@@ -1831,6 +1874,7 @@ def get_kv_cache_config_from_groups(
|
||||
layer_stride=layer_stride,
|
||||
block_stride=block_stride,
|
||||
offset=offset,
|
||||
slots_per_block=slots,
|
||||
)
|
||||
)
|
||||
byte_offset += len(layer_names) * spec.page_size_bytes
|
||||
@@ -2698,6 +2742,289 @@ def _project_kv_cache_groups_to_worker(
|
||||
return projected_groups
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# v2.5 side-cache layout (VLLM_E2_SIDE_CACHE_LAYOUT, default ON).
|
||||
#
|
||||
# The packed block-outermost layout strides every pool block by the widest
|
||||
# group, and every group draws whole blocks from the one pool. Flash-Next has
|
||||
# two side caches whose real state is tiny next to that stride (12.9 MB on the
|
||||
# binding PP stage): the TP-replicated PLE short-conv state (one layer, 2 + K
|
||||
# blocks per request in align mode, a 1.6 MB padded page) and the QSA raw-key
|
||||
# ring (one block per request, 16 KB for all of a stage's layers). Together
|
||||
# they hold 6 of the 103 blocks a 262K request needs. Two layout-only changes
|
||||
# remove that cost without changing any value a kernel reads:
|
||||
#
|
||||
# 1. PLE fold. The replicated Mamba group joins a sharded Mamba group with the
|
||||
# same scheduling parameters whose block has a spare page on every worker
|
||||
# (Flash-Next: a GDN group holds 6 of PP0's 7 pages), so the PLE state rides
|
||||
# in that group's blocks under the block table it would have had on its
|
||||
# own. No worker's stride changes, so the pool block count does not change.
|
||||
# 2. Ring slot packing. The ring group splits every pool block into S equal
|
||||
# slots and the ring manager hands out slots; a pool block goes back to the
|
||||
# pool when its last slot is released. The ring then costs cdiv(requests, S)
|
||||
# blocks in all instead of one per request.
|
||||
#
|
||||
# Off (=0) restores one group each, block for block.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Slot stride alignment for packed ring views (bytes) and the largest S tried.
|
||||
_RING_SLOT_ALIGN = 256
|
||||
_RING_MAX_SLOTS = 1024
|
||||
|
||||
|
||||
def _side_cache_layout_enabled(vllm_config: VllmConfig) -> bool:
|
||||
"""Whether the v2.5 side-cache layout may apply to this engine.
|
||||
|
||||
KV connectors (NIXL, offload, P/D) address groups and blocks themselves
|
||||
(the NIXL worker requires exactly one single-layer replicated group), and
|
||||
HiSparse has its own pool layout, so those engines keep the old layout.
|
||||
"""
|
||||
if not envs.VLLM_E2_SIDE_CACHE_LAYOUT:
|
||||
return False
|
||||
if getattr(vllm_config, "kv_transfer_config", None) is not None:
|
||||
return False
|
||||
attention_config = getattr(vllm_config, "attention_config", None)
|
||||
if getattr(attention_config, "hisparse_config", None) is not None:
|
||||
return False
|
||||
parallel_config = getattr(vllm_config, "parallel_config", None)
|
||||
if getattr(parallel_config, "decode_context_parallel_size", 1) not in (1, None):
|
||||
return False
|
||||
try:
|
||||
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
|
||||
except ValueError:
|
||||
# Unresolved layout: only reachable from callers that bypass the
|
||||
# engine core; nothing to pack without a block-outermost layout.
|
||||
return False
|
||||
return bool(layout.is_block_outermost)
|
||||
|
||||
|
||||
def _group_layer_specs(group: KVCacheGroupSpec) -> list[KVCacheSpec]:
|
||||
spec = group.kv_cache_spec
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
return [spec.kv_cache_specs[name] for name in group.layer_names]
|
||||
return [spec] * len(group.layer_names)
|
||||
|
||||
|
||||
def _mamba_schedule_key(spec: MambaSpec) -> tuple:
|
||||
"""Everything the scheduler's MambaManager and the worker's align-mode
|
||||
bookkeeping read from a group's spec. Two Mamba groups with equal keys get
|
||||
identical block tables for every request."""
|
||||
return (
|
||||
spec.block_size,
|
||||
spec.page_size_bytes,
|
||||
spec.num_speculative_blocks,
|
||||
spec.num_prefill_checkpoint_blocks,
|
||||
spec.prefill_checkpoint_alignment,
|
||||
spec.mamba_cache_mode,
|
||||
)
|
||||
|
||||
|
||||
def _worker_strides(
|
||||
groups: list[KVCacheGroupSpec], worker_specs: list[dict[str, KVCacheSpec]]
|
||||
) -> list[int]:
|
||||
"""Pool block bytes of every worker for this global grouping."""
|
||||
strides = []
|
||||
for spec in worker_specs:
|
||||
projected = _project_kv_cache_groups_to_worker(groups, spec)
|
||||
if not any(g.layer_names for g in projected):
|
||||
strides.append(0)
|
||||
continue
|
||||
strides.append(_get_kv_cache_bytes_per_block(projected))
|
||||
return strides
|
||||
|
||||
|
||||
def _fold_replicated_mamba_groups(
|
||||
groups: list[KVCacheGroupSpec],
|
||||
worker_specs: list[dict[str, KVCacheSpec]],
|
||||
) -> tuple[list[KVCacheGroupSpec], list[str]]:
|
||||
"""Fold each TP-replicated Mamba group into a compatible sharded one.
|
||||
|
||||
A target holds only sharded Mamba layers with the same scheduling key and
|
||||
EAGLE flag, and the merged group must not widen the block of any worker
|
||||
(checked on every worker's projection, so PP stages are covered). Among
|
||||
valid targets the lowest index wins. Returns the new group list and one
|
||||
log note per replicated group.
|
||||
"""
|
||||
groups = list(groups)
|
||||
notes: list[str] = []
|
||||
base_strides = _worker_strides(groups, worker_specs)
|
||||
index = 0
|
||||
while index < len(groups):
|
||||
source = groups[index]
|
||||
layer_specs = _group_layer_specs(source)
|
||||
if not (
|
||||
layer_specs
|
||||
and all(
|
||||
isinstance(spec, MambaSpec) and spec.tp_replicated
|
||||
for spec in layer_specs
|
||||
)
|
||||
and len({_mamba_schedule_key(spec) for spec in layer_specs}) == 1
|
||||
):
|
||||
index += 1
|
||||
continue
|
||||
key = _mamba_schedule_key(cast(MambaSpec, layer_specs[0]))
|
||||
chosen = None
|
||||
for target_index, target in enumerate(groups):
|
||||
if target_index == index or target.is_eagle_group != source.is_eagle_group:
|
||||
continue
|
||||
target_specs = _group_layer_specs(target)
|
||||
if not target_specs or not all(
|
||||
isinstance(spec, MambaSpec)
|
||||
and not spec.tp_replicated
|
||||
and _mamba_schedule_key(spec) == key
|
||||
for spec in target_specs
|
||||
):
|
||||
continue
|
||||
merged_specs: dict[str, KVCacheSpec] = dict(
|
||||
zip(target.layer_names, target_specs)
|
||||
)
|
||||
merged_specs.update(zip(source.layer_names, layer_specs))
|
||||
# Built directly: is_uniform_type rejects mixed tp_replicated on
|
||||
# purpose (NIXL), and connectors never reach this path.
|
||||
merged = KVCacheGroupSpec(
|
||||
target.layer_names + source.layer_names,
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=target_specs[0].block_size,
|
||||
kv_cache_specs=merged_specs,
|
||||
),
|
||||
is_eagle_group=target.is_eagle_group,
|
||||
)
|
||||
trial = [
|
||||
merged if i == target_index else g
|
||||
for i, g in enumerate(groups)
|
||||
if i != index
|
||||
]
|
||||
strides = _worker_strides(trial, worker_specs)
|
||||
if all(new <= old for new, old in zip(strides, base_strides)):
|
||||
chosen = (target_index, merged, trial)
|
||||
break
|
||||
if chosen is None:
|
||||
notes.append(
|
||||
f"replicated Mamba group {index} ({len(source.layer_names)} "
|
||||
"layer(s)) kept separate: no sharded group has a spare page on "
|
||||
"every worker"
|
||||
)
|
||||
index += 1
|
||||
continue
|
||||
target_index, merged, trial = chosen
|
||||
notes.append(
|
||||
f"folded replicated Mamba group {index} "
|
||||
f"({', '.join(source.layer_names)}) into Mamba group "
|
||||
f"{target_index} (now {len(merged.layer_names)} layers); per-worker "
|
||||
f"block bytes unchanged {sorted(set(base_strides))}"
|
||||
)
|
||||
groups = trial
|
||||
# The group after the folded one now sits at `index`.
|
||||
return groups, notes
|
||||
|
||||
|
||||
def _is_ring_group(group: KVCacheGroupSpec) -> bool:
|
||||
spec = group.kv_cache_spec
|
||||
layer_specs = (
|
||||
list(spec.kv_cache_specs.values())
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs)
|
||||
else [spec]
|
||||
)
|
||||
return bool(layer_specs) and all(
|
||||
type(layer_spec) is CircularBufferSpec for layer_spec in layer_specs
|
||||
)
|
||||
|
||||
|
||||
def _choose_ring_slots(
|
||||
projected_groups_per_worker: list[list[KVCacheGroupSpec]],
|
||||
) -> dict[int, int]:
|
||||
"""Slots per pool block for every ring group (group index -> S >= 2).
|
||||
|
||||
S must cut every worker's block stride into equal slots that are
|
||||
`_RING_SLOT_ALIGN`-aligned and hold that worker's ring pages; the largest
|
||||
such S up to `_RING_MAX_SLOTS` is taken. Block ids are global, so S is the
|
||||
same on every worker. A ring group with no valid S keeps one pool block
|
||||
per request.
|
||||
"""
|
||||
chosen: dict[int, int] = {}
|
||||
if not projected_groups_per_worker or not projected_groups_per_worker[0]:
|
||||
return chosen
|
||||
num_groups = len(projected_groups_per_worker[0])
|
||||
strides = [
|
||||
_get_kv_cache_bytes_per_block(groups)
|
||||
if any(g.layer_names for g in groups)
|
||||
else 0
|
||||
for groups in projected_groups_per_worker
|
||||
]
|
||||
for group_index in range(num_groups):
|
||||
if not _is_ring_group(projected_groups_per_worker[0][group_index]):
|
||||
continue
|
||||
needs = []
|
||||
for groups, stride in zip(projected_groups_per_worker, strides):
|
||||
group = groups[group_index]
|
||||
if not group.layer_names:
|
||||
continue
|
||||
ring_bytes = sum(spec.page_size_bytes for spec in _group_layer_specs(group))
|
||||
needs.append((stride, ring_bytes))
|
||||
if not needs:
|
||||
continue
|
||||
for slots in range(_RING_MAX_SLOTS, 1, -1):
|
||||
if all(
|
||||
stride % slots == 0
|
||||
and (stride // slots) % _RING_SLOT_ALIGN == 0
|
||||
and stride // slots >= ring_bytes
|
||||
for stride, ring_bytes in needs
|
||||
):
|
||||
chosen[group_index] = slots
|
||||
break
|
||||
return chosen
|
||||
|
||||
|
||||
def _apply_side_cache_layout(
|
||||
vllm_config: VllmConfig,
|
||||
global_kv_cache_groups: list[KVCacheGroupSpec],
|
||||
kv_cache_specs: list[dict[str, KVCacheSpec]],
|
||||
) -> tuple[list[KVCacheGroupSpec], list[list[KVCacheGroupSpec]]]:
|
||||
"""Fold side Mamba groups, project per worker, then pick ring slots.
|
||||
|
||||
Returns the (possibly folded) global groups and every worker's projection,
|
||||
with `slots_per_block` set on packed ring groups.
|
||||
"""
|
||||
relevant = any(
|
||||
(isinstance(spec, MambaSpec) and spec.tp_replicated)
|
||||
or type(spec) is CircularBufferSpec
|
||||
for worker_spec in kv_cache_specs
|
||||
for spec in worker_spec.values()
|
||||
)
|
||||
enabled = relevant and _side_cache_layout_enabled(vllm_config)
|
||||
notes: list[str] = []
|
||||
if enabled:
|
||||
global_kv_cache_groups, notes = _fold_replicated_mamba_groups(
|
||||
global_kv_cache_groups, kv_cache_specs
|
||||
)
|
||||
projected = [
|
||||
_project_kv_cache_groups_to_worker(global_kv_cache_groups, worker_spec)
|
||||
for worker_spec in kv_cache_specs
|
||||
]
|
||||
if enabled and all(
|
||||
_glm5_next_tensor_layout(groups) is None for groups in projected if groups
|
||||
):
|
||||
for group_index, slots in _choose_ring_slots(projected).items():
|
||||
for groups in projected:
|
||||
groups[group_index].slots_per_block = slots
|
||||
notes.append(
|
||||
f"QSA ring group {group_index} packs {slots} requests per pool block"
|
||||
)
|
||||
if relevant and envs.VLLM_E2_SIDE_CACHE_LAYOUT and not enabled:
|
||||
notes.append(
|
||||
"not applicable to this engine (KV connector, HiSparse, DCP or a "
|
||||
"layer-outermost layout)"
|
||||
)
|
||||
if relevant:
|
||||
# The A/B evidence line: boot logs must show which arm ran.
|
||||
logger.info(
|
||||
"Side-cache layout: %s (VLLM_E2_SIDE_CACHE_LAYOUT)%s",
|
||||
"on" if enabled else "off",
|
||||
"".join(f"; {note}" for note in notes),
|
||||
)
|
||||
return global_kv_cache_groups, projected
|
||||
|
||||
|
||||
def get_kv_cache_configs(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_specs: list[dict[str, KVCacheSpec]],
|
||||
@@ -2767,10 +3094,11 @@ def get_kv_cache_configs(
|
||||
# If original_max_model_len was -1, automatically
|
||||
# determine the maximum model length that fits in available GPU memory.
|
||||
# We use per-worker projected groups to account for PP sharding.
|
||||
projected_groups_per_worker = [
|
||||
_project_kv_cache_groups_to_worker(global_kv_cache_groups, worker_spec)
|
||||
for worker_spec in kv_cache_specs
|
||||
]
|
||||
# v2.5: the side-cache layout (PLE fold, ring slots) is applied here, where
|
||||
# every worker's layers are known, so no PP stage's block can widen.
|
||||
global_kv_cache_groups, projected_groups_per_worker = _apply_side_cache_layout(
|
||||
vllm_config, global_kv_cache_groups, kv_cache_specs
|
||||
)
|
||||
|
||||
# If `num_gpu_blocks_override` is set, the cache size that will actually
|
||||
# be allocated is decoupled from the profiled `available_memory`:
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Debug-only scheduler allocation log for KV-capacity probes (off unless
|
||||
VLLM_E2_SCHED_LOG=<path>).
|
||||
|
||||
One JSON line per scheduler update while any request exists, and one "fin" line
|
||||
per request as it is freed:
|
||||
{"t": wall, "m": monotonic, "step": n, "total": pool blocks,
|
||||
"free": free blocks, "running": r, "waiting": w,
|
||||
"tot": tokens scheduled this step,
|
||||
"reqs": [[id, prompt, computed, output, cached, preemptions,
|
||||
spec_scheduled, status, [blocks per group],
|
||||
tokens scheduled for this request this step]]}
|
||||
{"t": wall, "m": monotonic,
|
||||
"fin": [id, prompt, computed, output, cached, preemptions, status,
|
||||
[blocks per group]]}
|
||||
Blocks are counted per KV-cache group, null blocks excluded. A logging failure
|
||||
writes an "error" line and never raises into the scheduler. Unset = the
|
||||
scheduler only pays one env check per update.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
_PATH = os.environ.get("VLLM_E2_SCHED_LOG", "")
|
||||
_state = {
|
||||
"f": None,
|
||||
"step": 0,
|
||||
"flush": 0.0,
|
||||
"wrapped": False,
|
||||
"errors": 0,
|
||||
"checked": False,
|
||||
}
|
||||
REQ_FIELDS = (
|
||||
"request_id",
|
||||
"num_prompt_tokens",
|
||||
"num_computed_tokens",
|
||||
"num_output_tokens",
|
||||
"num_preemptions",
|
||||
"status",
|
||||
"prefill_stats",
|
||||
)
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
return bool(_PATH)
|
||||
|
||||
|
||||
def _blocks(sched, rid):
|
||||
return [
|
||||
sum(1 for b in m.req_to_blocks.get(rid, ()) if not b.is_null)
|
||||
for m in sched.kv_cache_manager.coordinator.single_type_managers
|
||||
]
|
||||
|
||||
|
||||
def _status(r):
|
||||
return getattr(r.status, "name", str(r.status))
|
||||
|
||||
|
||||
def _cached(r):
|
||||
"""Prefix-cache hit for this request. vLLM keeps it in
|
||||
`request.prefill_stats` (PrefillStats.num_cached_tokens = local +
|
||||
external); the output processor takes those stats after the first prefill
|
||||
(`take_prefill_stats` sets the attribute to None), so -1 means "no longer
|
||||
knowable from the scheduler", never a silent zero."""
|
||||
ps = getattr(r, "prefill_stats", None)
|
||||
return int(getattr(ps, "num_cached_tokens", -1)) if ps is not None else -1
|
||||
|
||||
|
||||
def _contract(r):
|
||||
"""Once per process: every field this log reads must exist on the real Request
|
||||
object. A missing field is written once as a schema_error record — the evaluator
|
||||
fails closed on it — instead of turning every row into an error."""
|
||||
if _state["checked"]:
|
||||
return
|
||||
_state["checked"] = True
|
||||
missing = [f for f in REQ_FIELDS if not hasattr(r, f)]
|
||||
if missing:
|
||||
_write(
|
||||
{
|
||||
"t": round(time.time(), 4),
|
||||
"m": round(time.monotonic(), 4),
|
||||
"schema_error": missing,
|
||||
},
|
||||
force=True,
|
||||
)
|
||||
|
||||
|
||||
def _write(rec, force=False):
|
||||
f = _state["f"]
|
||||
if f is None:
|
||||
# noqa SIM115: one append handle for the process lifetime, kept in
|
||||
# _state and flushed on a cadence; a context manager would reopen the
|
||||
# log on every scheduler update.
|
||||
f = _state["f"] = open(_PATH, "a", buffering=1 << 16) # noqa: SIM115
|
||||
f.write(json.dumps(rec, separators=(",", ":")) + "\n")
|
||||
if force or rec["t"] - _state["flush"] > 0.5:
|
||||
f.flush()
|
||||
_state["flush"] = rec["t"]
|
||||
|
||||
|
||||
def _error(e):
|
||||
_state["errors"] += 1
|
||||
if _state["errors"] <= 20:
|
||||
try: # noqa: SIM105
|
||||
_write(
|
||||
{
|
||||
"t": round(time.time(), 4),
|
||||
"m": round(time.monotonic(), 4),
|
||||
"error": f"{type(e).__name__}: {e}"[:300],
|
||||
},
|
||||
force=True,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
# SIM105: this module imports json, os and time only, so that an
|
||||
# enabled log costs the scheduler nothing it can see; the
|
||||
# last-resort writer does not earn a contextlib import.
|
||||
pass
|
||||
|
||||
|
||||
def _wrap_free(sched):
|
||||
orig = sched._free_request
|
||||
|
||||
def _free_request(request, *args, **kwargs):
|
||||
try:
|
||||
_write(
|
||||
{
|
||||
"t": round(time.time(), 4),
|
||||
"m": round(time.monotonic(), 4),
|
||||
"fin": [
|
||||
request.request_id,
|
||||
request.num_prompt_tokens,
|
||||
request.num_computed_tokens,
|
||||
request.num_output_tokens,
|
||||
_cached(request),
|
||||
request.num_preemptions,
|
||||
_status(request),
|
||||
_blocks(sched, request.request_id),
|
||||
],
|
||||
},
|
||||
force=True,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 - the debug log must never break the scheduler
|
||||
_error(e)
|
||||
return orig(request, *args, **kwargs)
|
||||
|
||||
sched._free_request = _free_request
|
||||
_state["wrapped"] = True
|
||||
|
||||
|
||||
def log_step(sched, scheduler_output) -> None:
|
||||
try:
|
||||
if not _state["wrapped"]:
|
||||
_wrap_free(sched)
|
||||
if not sched.requests:
|
||||
return
|
||||
_state["step"] += 1
|
||||
spec = getattr(scheduler_output, "scheduled_spec_decode_tokens", None) or {}
|
||||
pool = sched.kv_cache_manager.block_pool
|
||||
nsched = getattr(scheduler_output, "num_scheduled_tokens", None) or {}
|
||||
_contract(next(iter(sched.requests.values())))
|
||||
rows = [
|
||||
[
|
||||
rid,
|
||||
r.num_prompt_tokens,
|
||||
r.num_computed_tokens,
|
||||
r.num_output_tokens,
|
||||
_cached(r),
|
||||
r.num_preemptions,
|
||||
len(spec.get(rid, ())),
|
||||
_status(r),
|
||||
_blocks(sched, rid),
|
||||
int(nsched.get(rid, 0)),
|
||||
]
|
||||
for rid, r in sched.requests.items()
|
||||
]
|
||||
_write(
|
||||
{
|
||||
"t": round(time.time(), 4),
|
||||
"m": round(time.monotonic(), 4),
|
||||
"step": _state["step"],
|
||||
"total": pool.num_gpu_blocks,
|
||||
"free": pool.get_num_free_blocks(),
|
||||
"running": len(sched.running),
|
||||
"waiting": len(sched.waiting),
|
||||
"tot": int(
|
||||
getattr(scheduler_output, "total_num_scheduled_tokens", 0) or 0
|
||||
),
|
||||
"reqs": rows,
|
||||
}
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
_error(e)
|
||||
@@ -301,6 +301,13 @@ class SchedulerOutput:
|
||||
# preventing stale NaN/data from corrupting attention or SSM computation.
|
||||
new_block_ids_to_zero: list[int] | None = None
|
||||
|
||||
# v2.2 ring-row tag fix: ring (CircularBufferSpec) blocks claimed during this
|
||||
# scheduling step. Independent of new_block_ids_to_zero and of
|
||||
# needs_kv_cache_zeroing: the worker clears only the rows' tag words, after
|
||||
# the zeroer and before any copy or forward of this step, i.e. before the
|
||||
# new owner's first read or write of the ring.
|
||||
new_ring_block_ids_to_clear: list[int] | None = None
|
||||
|
||||
# CoW copies to apply after zeroing new blocks and before forward.
|
||||
kv_cache_block_copies: list[KVCacheBlockCopy] | None = None
|
||||
|
||||
@@ -314,6 +321,10 @@ class SchedulerOutput:
|
||||
# Number of spec tokens to schedule for the next step.
|
||||
num_spec_tokens_to_schedule: int = 0
|
||||
|
||||
# Identity of this schedule() call (`Scheduler.current_step`). Used to
|
||||
# retrieve the matching consumption-time draft-token snapshot.
|
||||
scheduler_step: int = 0
|
||||
|
||||
@classmethod
|
||||
def make_empty(cls) -> "SchedulerOutput":
|
||||
return cls(
|
||||
@@ -338,5 +349,10 @@ class GrammarOutput:
|
||||
# Per request, ordered as structured_output_request_ids: how many leading
|
||||
# drafts the bitmask constrained. `grammar_bitmask` fills every row after
|
||||
# the first -1 placeholder with the all-permissive `_full_mask`, so drafts
|
||||
# from this index on must not be accepted. None means nothing to invalidate.
|
||||
num_acceptable_drafts: list[int] | None = None
|
||||
# Per request, ordered as structured_output_request_ids: how many leading
|
||||
# drafts the bitmask constrained. `grammar_bitmask` fills every row after
|
||||
# the first -1 placeholder with the all-permissive `_full_mask`, so drafts
|
||||
# from this index on must not be accepted. None invalidates every draft.
|
||||
num_acceptable_drafts: list[int] | None = None
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import itertools
|
||||
import os
|
||||
import time
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Iterable
|
||||
@@ -38,6 +39,7 @@ from vllm.v1.core.encoder_cache_manager import (
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
||||
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlock
|
||||
from vllm.v1.core.sched import e2_alloc_log as _e2_alloc_log
|
||||
from vllm.v1.core.sched.interface import PauseState, SchedulerInterface
|
||||
from vllm.v1.core.sched.output import (
|
||||
CachedRequestData,
|
||||
@@ -55,10 +57,12 @@ from vllm.v1.core.sched.request_queue import (
|
||||
from vllm.v1.core.sched.utils import check_stop, remove_all
|
||||
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
KVCacheConfig,
|
||||
MambaSpec,
|
||||
get_mamba_prefill_checkpoint_position,
|
||||
is_mamba_prefill_checkpoint_valid,
|
||||
iter_layer_specs,
|
||||
)
|
||||
from vllm.v1.metrics.perf import ModelMetrics, PerfStats
|
||||
from vllm.v1.metrics.stats import (
|
||||
@@ -76,10 +80,15 @@ from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
|
||||
from vllm.v1.spec_decode.metrics import SpecDecodingStats
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
from vllm.v1.structured_output.utils import strip_speculative_padding
|
||||
from vllm.v1.structured_output.utils import strip_speculative_padding
|
||||
from vllm.v1.utils import record_function_or_nullcontext
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Bounds-guard instrumentation: verify KVBlockZeroer's exclusive-ownership precondition before a
|
||||
# block is zeroed. Off by default, so it costs nothing.
|
||||
_ZERO_OWNERSHIP_CHECK = os.getenv("VLLM_E2_ZERO_OWNERSHIP_CHECK") == "1"
|
||||
|
||||
|
||||
class Scheduler(SchedulerInterface):
|
||||
def __init__(
|
||||
@@ -187,6 +196,26 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
self.requires_kv_delivery = self.connector.requires_kv_delivery
|
||||
|
||||
# Ring-skip telemetry: the QSA ring block (CircularBufferSpec) is rewritten
|
||||
# by every step that touches the request, so with overlapping batches
|
||||
# (async scheduling or PP) an in-flight step may still be committing a
|
||||
# freed/preempted request's suffix rows when the scheduler hands that
|
||||
# block to another request. A KV connector is not the only writer that
|
||||
# can race the stale in-flight write; the next request's own first
|
||||
# step is. Defer the return of freed blocks to the pool until the
|
||||
# newest scheduled step's output has been processed whenever a ring
|
||||
# group exists. Same mechanism and fence as the connector gate above.
|
||||
if (
|
||||
not self.defer_block_free
|
||||
and self.vllm_config.max_concurrent_batches > 1
|
||||
and any(
|
||||
isinstance(spec, CircularBufferSpec)
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||
)
|
||||
):
|
||||
self.defer_block_free = True
|
||||
|
||||
self.kv_event_publisher = EventPublisherFactory.create(
|
||||
self.kv_events_config,
|
||||
self.parallel_config.data_parallel_index,
|
||||
@@ -337,6 +366,16 @@ class Scheduler(SchedulerInterface):
|
||||
# Bind after construction so connectors can access the cache manager.
|
||||
if self.connector is not None:
|
||||
self.connector.bind_kv_cache_manager(self.kv_cache_manager)
|
||||
# Finished turns' grace pins are off with a connector, as
|
||||
# waiter pins are (_pin_waiting_hits).
|
||||
self.kv_cache_manager.enable_grace_pins = False
|
||||
# Ring blocks an async KV load is about to fill. Their tag clear
|
||||
# is skipped for the same reason as zeroing (it could land after the
|
||||
# out-of-band write and wipe the producer's committed tags). Tracked
|
||||
# whatever needs_kv_cache_zeroing says, like the clear itself.
|
||||
self._skip_ring_clear_block_ids: set[int] = set()
|
||||
|
||||
self._log_prefix_cache_features()
|
||||
|
||||
self.use_pp = self.parallel_config.pipeline_parallel_size > 1
|
||||
self.use_v2_model_runner = vllm_config.use_v2_model_runner
|
||||
@@ -535,6 +574,54 @@ class Scheduler(SchedulerInterface):
|
||||
end = min((s for s in stops if start < s < end), default=end)
|
||||
return max(end - start, 0)
|
||||
|
||||
def _log_prefix_cache_features(self) -> None:
|
||||
"""One startup line per prefix-cache retention feature that is active
|
||||
(nothing when it is off)."""
|
||||
manager = self.kv_cache_manager
|
||||
if manager.enable_caching and self.connector is None:
|
||||
logger.info(
|
||||
"Prefix cache: waiting requests' cached prefixes are pinned "
|
||||
"until admission (up to %d waiters; hash unit %d, block %d)",
|
||||
manager.MAX_PINNED_WAITERS,
|
||||
self.hash_block_size,
|
||||
self.block_size,
|
||||
)
|
||||
if manager.enable_grace_pins:
|
||||
logger.info(
|
||||
"Prefix cache: a finished turn's cached prefix stays referenced "
|
||||
"until its next turn is admitted (up to %d turns; hash unit %d, "
|
||||
"block %d)",
|
||||
manager.MAX_GRACE_PINS,
|
||||
self.hash_block_size,
|
||||
self.block_size,
|
||||
)
|
||||
|
||||
def _pin_waiting_hits(self) -> None:
|
||||
"""Pin the prefix-cache hits of the requests at the front of the
|
||||
waiting queue (arrival order, bounded by the manager) so the
|
||||
admissions and decode growth of other requests cannot evict them
|
||||
while they wait -- also while the running set is full, when they are
|
||||
not even looked up. Released on admission, free, and whenever a
|
||||
running request would otherwise fail to allocate.
|
||||
"""
|
||||
manager = self.kv_cache_manager
|
||||
if (
|
||||
not manager.enable_caching
|
||||
or self.connector is not None
|
||||
or manager.num_pinned_waiters >= manager.MAX_PINNED_WAITERS
|
||||
):
|
||||
return
|
||||
for request in self.waiting:
|
||||
if manager.num_pinned_waiters >= manager.MAX_PINNED_WAITERS:
|
||||
break
|
||||
if (
|
||||
request.num_computed_tokens != 0
|
||||
or request.status
|
||||
not in (RequestStatus.WAITING, RequestStatus.PREEMPTED)
|
||||
):
|
||||
continue
|
||||
manager.pin_waiting_hit(request)
|
||||
|
||||
def _get_local_prefix_cache_hit(
|
||||
self, request: Request
|
||||
) -> tuple[KVCacheBlocks, int, int, bool]:
|
||||
@@ -769,6 +856,12 @@ class Scheduler(SchedulerInterface):
|
||||
):
|
||||
break
|
||||
|
||||
# Waiting requests' pinned prefix hits yield
|
||||
# before anyone is preempted (a pin never costs a running
|
||||
# request its blocks); retry with them in the free queue.
|
||||
if self.kv_cache_manager.yield_pinned_hits(limit=1):
|
||||
continue
|
||||
|
||||
# The request cannot be scheduled.
|
||||
# Preempt the lowest-priority request.
|
||||
if self.policy == SchedulingPolicy.PRIORITY:
|
||||
@@ -780,6 +873,13 @@ class Scheduler(SchedulerInterface):
|
||||
preempted_req = self.running[-1]
|
||||
|
||||
# A deferred free will not help with immediate allocation.
|
||||
# (Upstream #49675. It subsumes our earlier post-hoc guard,
|
||||
# which stopped after ONE deferred victim: a victim that
|
||||
# passes this check is freed immediately by
|
||||
# _preempt_request, so deferred_frees cannot grow below.
|
||||
# The requester stays running, unscheduled, and retries
|
||||
# after the fence drains; with every running request in
|
||||
# flight (PP / async) that is zero victims, not the batch.)
|
||||
if not self._request_blocks_can_be_freed(preempted_req):
|
||||
break
|
||||
|
||||
@@ -1238,6 +1338,103 @@ class Scheduler(SchedulerInterface):
|
||||
has_scheduled_reqs=bool(self.running),
|
||||
)
|
||||
|
||||
# Finished turns' grace pins (oldest first), then the
|
||||
# pins of waiters behind this one, yield to it one at a time
|
||||
# (arrival order). Its own hit is pinned first (a turn that
|
||||
# arrived this step is not pinned yet and its hit may sit in
|
||||
# a grace that yields), so only blocks it does not resume
|
||||
# from go back. And only when releasing them all would cover
|
||||
# the shortfall: otherwise yielding costs other conversations
|
||||
# their prefix and still does not admit this one.
|
||||
kv_cache_manager = self.kv_cache_manager
|
||||
yield_for_head = new_blocks is None and (
|
||||
kv_cache_manager.last_alloc_shortfall > 0
|
||||
and (
|
||||
kv_cache_manager.num_grace_pins > 0
|
||||
or kv_cache_manager.num_pinned_waiters > 0
|
||||
)
|
||||
)
|
||||
if yield_for_head:
|
||||
if (
|
||||
self.connector is None
|
||||
and request.num_computed_tokens == 0
|
||||
and kv_cache_manager.pin_waiting_hit(request) > 0
|
||||
):
|
||||
# The pin just took blocks out of the free queue that
|
||||
# the failed attempt counted as free: the proof block
|
||||
# past the hit, and any hit block the lookup above did
|
||||
# not return. Recompute the shortfall on the pinned
|
||||
# state before measuring it against the others' pins.
|
||||
new_blocks = self.kv_cache_manager.allocate_slots(
|
||||
request,
|
||||
num_new_tokens,
|
||||
num_new_computed_tokens=num_new_local_computed_tokens,
|
||||
new_computed_blocks=new_computed_blocks,
|
||||
num_lookahead_tokens=effective_lookahead_tokens,
|
||||
num_external_computed_tokens=num_external_computed_tokens,
|
||||
delay_cache_blocks=load_kv_async,
|
||||
num_encoder_tokens=num_encoder_tokens,
|
||||
full_sequence_must_fit=self.scheduler_reserve_full_isl,
|
||||
reserved_blocks=reserved_blocks,
|
||||
has_scheduled_reqs=bool(self.running),
|
||||
)
|
||||
yield_for_head = new_blocks is None and (
|
||||
kv_cache_manager.last_alloc_shortfall
|
||||
<= kv_cache_manager.num_yieldable_blocks(
|
||||
keep=request_id, keep_hit=new_computed_blocks.blocks
|
||||
)
|
||||
)
|
||||
if (
|
||||
yield_for_head
|
||||
and kv_cache_manager.release_pinned_extras(
|
||||
request_id, new_computed_blocks.blocks
|
||||
)
|
||||
> 0
|
||||
):
|
||||
# Its own pinned blocks outside the hit it is admitted
|
||||
# on (the proof block past it) go first: they cost
|
||||
# nobody else anything, and at an exact fit they are
|
||||
# the blocks it needs. Retry on them before the others
|
||||
# yield.
|
||||
new_blocks = self.kv_cache_manager.allocate_slots(
|
||||
request,
|
||||
num_new_tokens,
|
||||
num_new_computed_tokens=num_new_local_computed_tokens,
|
||||
new_computed_blocks=new_computed_blocks,
|
||||
num_lookahead_tokens=effective_lookahead_tokens,
|
||||
num_external_computed_tokens=num_external_computed_tokens,
|
||||
delay_cache_blocks=load_kv_async,
|
||||
num_encoder_tokens=num_encoder_tokens,
|
||||
full_sequence_must_fit=self.scheduler_reserve_full_isl,
|
||||
reserved_blocks=reserved_blocks,
|
||||
has_scheduled_reqs=bool(self.running),
|
||||
)
|
||||
# Coverage is re-read before every yield: a retry can fail
|
||||
# by more than the attempt before it (the full-sequence gate
|
||||
# passes, the lookahead gate does not).
|
||||
while (
|
||||
yield_for_head
|
||||
and new_blocks is None
|
||||
and kv_cache_manager.last_alloc_shortfall
|
||||
<= kv_cache_manager.num_yieldable_blocks(
|
||||
keep=request_id, keep_hit=new_computed_blocks.blocks
|
||||
)
|
||||
and kv_cache_manager.yield_pinned_hits(keep=request_id, limit=1)
|
||||
):
|
||||
new_blocks = self.kv_cache_manager.allocate_slots(
|
||||
request,
|
||||
num_new_tokens,
|
||||
num_new_computed_tokens=num_new_local_computed_tokens,
|
||||
new_computed_blocks=new_computed_blocks,
|
||||
num_lookahead_tokens=effective_lookahead_tokens,
|
||||
num_external_computed_tokens=num_external_computed_tokens,
|
||||
delay_cache_blocks=load_kv_async,
|
||||
num_encoder_tokens=num_encoder_tokens,
|
||||
full_sequence_must_fit=self.scheduler_reserve_full_isl,
|
||||
reserved_blocks=reserved_blocks,
|
||||
has_scheduled_reqs=bool(self.running),
|
||||
)
|
||||
|
||||
if new_blocks is None:
|
||||
# The request cannot be scheduled.
|
||||
|
||||
@@ -1293,6 +1490,12 @@ class Scheduler(SchedulerInterface):
|
||||
request.num_computed_tokens = num_computed_tokens
|
||||
self._inflight_prefills.add(request)
|
||||
self._set_kv_fetch_stage(request, KV_FETCH_IN_PROGRESS)
|
||||
# Same race for the ring's tag clear; the load
|
||||
# brings the producer's rows and tags (or none, and the
|
||||
# reader then fails closed on the missing history).
|
||||
self._skip_ring_clear_block_ids.update(
|
||||
self.kv_cache_manager.get_ring_block_ids(request.request_id)
|
||||
)
|
||||
skip_request(request_queue)
|
||||
continue
|
||||
|
||||
@@ -1354,6 +1557,9 @@ class Scheduler(SchedulerInterface):
|
||||
if step_skipped_waiting:
|
||||
self.waiting.prepend_requests(step_skipped_waiting)
|
||||
|
||||
# Whoever is still waiting keeps its prefix-cache hit.
|
||||
self._pin_waiting_hits()
|
||||
|
||||
# DP prefill balancing: on a step that admitted prefills (release),
|
||||
# record whether it was capacity-bound.
|
||||
if not defer_prefills:
|
||||
@@ -1485,11 +1691,14 @@ class Scheduler(SchedulerInterface):
|
||||
finished_req_ids=self.finished_req_ids,
|
||||
free_encoder_mm_hashes=self.encoder_cache_manager.get_freed_mm_hashes(),
|
||||
new_block_ids_to_zero=self._get_new_block_ids_to_zero(),
|
||||
# Drained every step, never gated on KV zeroing.
|
||||
new_ring_block_ids_to_clear=self._get_new_ring_block_ids_to_clear(),
|
||||
has_sync_kv_loads=has_sync_kv_loads,
|
||||
kv_cache_block_copies=pending_kv_cache_block_copies,
|
||||
kv_connector_block_state=kv_connector_block_state,
|
||||
num_spec_tokens_to_schedule=num_spec_tokens_to_schedule,
|
||||
ec_manager_metadata=self.encoder_cache_manager.get_manager_metadata(),
|
||||
scheduler_step=self.current_step,
|
||||
)
|
||||
|
||||
# NOTE(Kuntai): this function is designed for multiple purposes:
|
||||
@@ -1531,14 +1740,62 @@ class Scheduler(SchedulerInterface):
|
||||
) -> KVConnectorMetadata:
|
||||
return connector.build_connector_meta(scheduler_output)
|
||||
|
||||
def _get_new_ring_block_ids_to_clear(self) -> list[int] | None:
|
||||
"""Ring claims of this step whose tag words the worker clears.
|
||||
Unconditional (not under needs_kv_cache_zeroing); only blocks an async
|
||||
KV load is about to write are left out."""
|
||||
ids = self.kv_cache_manager.take_new_ring_block_ids()
|
||||
if self._skip_ring_clear_block_ids:
|
||||
skip = self._skip_ring_clear_block_ids
|
||||
ids = [b for b in ids if b not in skip]
|
||||
skip.clear()
|
||||
return ids or None
|
||||
|
||||
def _get_new_block_ids_to_zero(self) -> list[int] | None:
|
||||
# Drain new attention block ids every step so the manager-side list
|
||||
# does not grow unbounded; only kv-cache zeroing consumes them.
|
||||
new_block_ids_to_zero = self.kv_cache_manager.take_new_block_ids()
|
||||
if not self.needs_kv_cache_zeroing:
|
||||
return None
|
||||
if new_block_ids_to_zero and _ZERO_OWNERSHIP_CHECK:
|
||||
self._check_zero_block_ownership(new_block_ids_to_zero)
|
||||
|
||||
return new_block_ids_to_zero or None
|
||||
|
||||
def _check_zero_block_ownership(self, block_ids: list[int]) -> None:
|
||||
"""Test KVBlockZeroer's stated precondition, host side, off by default.
|
||||
|
||||
worker/utils.py justifies zeroing a whole tile with "block IDs are
|
||||
global pool indices and a newly allocated block owns its whole tile".
|
||||
Ownership is visible only here, in the scheduler process, so the check
|
||||
lives here rather than in the zeroer. A block about to be zeroed should
|
||||
be referenced by exactly one live request. More than one means a stale
|
||||
reference survived into the zeroing, and the kernel is about to destroy
|
||||
state another request still points at -- the stale-reference shape.
|
||||
|
||||
VLLM_E2_ZERO_OWNERSHIP_CHECK=1. O(live blocks) per step.
|
||||
"""
|
||||
pending = set(block_ids)
|
||||
owners: dict[int, set[str]] = {}
|
||||
for mgr in self.kv_cache_manager.coordinator.single_type_managers:
|
||||
if getattr(mgr, "slot_pool", None) is not None:
|
||||
# v2.5 slot-packed ring: its block ids are slot ids, not pool
|
||||
# block ids, and its pages are never zeroed.
|
||||
continue
|
||||
for req_id, blocks in mgr.req_to_blocks.items():
|
||||
for blk in blocks:
|
||||
if blk.block_id in pending:
|
||||
owners.setdefault(blk.block_id, set()).add(req_id)
|
||||
for block_id, req_ids in sorted(owners.items()):
|
||||
if len(req_ids) > 1:
|
||||
logger.warning(
|
||||
"KV zeroing ownership violation: block %d is about to be "
|
||||
"zeroed but is referenced by %d live requests (%s).",
|
||||
block_id,
|
||||
len(req_ids),
|
||||
", ".join(sorted(req_ids)[:4]),
|
||||
)
|
||||
|
||||
def _preempt_request(
|
||||
self, request: Request, timestamp: float, drop_stale_output: bool = False
|
||||
) -> None:
|
||||
@@ -2373,6 +2630,8 @@ class Scheduler(SchedulerInterface):
|
||||
engine_core_outputs[0] = eco = EngineCoreOutputs()
|
||||
eco.scheduler_stats = stats
|
||||
|
||||
if _e2_alloc_log.enabled():
|
||||
_e2_alloc_log.log_step(self, scheduler_output)
|
||||
return engine_core_outputs
|
||||
|
||||
def _ec_transfer_pending(self, request: Request, num_computed_tokens: int) -> bool:
|
||||
@@ -2735,7 +2994,9 @@ class Scheduler(SchedulerInterface):
|
||||
if not self.defer_block_free or fence_seq <= self.processed_step_seq:
|
||||
self.kv_cache_manager.block_pool.free_blocks(blocks)
|
||||
return
|
||||
self.deferred_frees.append((fence_seq, blocks[::-1]))
|
||||
# The immediate path above frees these as-is, so enqueue them as-is:
|
||||
# the drain no longer reverses.
|
||||
self.deferred_frees.append((fence_seq, blocks))
|
||||
|
||||
def _drain_deferred_frees(self):
|
||||
"""Return deferred blocks whose fence step has completed.
|
||||
@@ -2749,8 +3010,9 @@ class Scheduler(SchedulerInterface):
|
||||
if fence > self.processed_step_seq:
|
||||
break
|
||||
_, blocks = self.deferred_frees.popleft()
|
||||
# Free in reverse order so that the tail blocks are evicted first.
|
||||
self.kv_cache_manager.block_pool.free_blocks(reversed(blocks))
|
||||
# Already in eviction order (reversed per KV cache group by
|
||||
# `pop_blocks_for_free`), so free as-is.
|
||||
self.kv_cache_manager.block_pool.free_blocks(blocks)
|
||||
|
||||
def get_num_unfinished_requests(self) -> int:
|
||||
if self._pause_state == PauseState.PAUSED_ALL:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,9 +3,10 @@
|
||||
import itertools
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Callable, Sequence
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from vllm import envs
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
@@ -121,6 +122,12 @@ class SingleTypeKVCacheManager(ABC):
|
||||
and not isinstance(kv_cache_spec, CircularBufferSpec)
|
||||
)
|
||||
self.new_block_ids: list[int] = []
|
||||
# v2.2 ring-row tag fix: ring blocks claimed since the last drain. The ring stays
|
||||
# OUT of whole-block zeroing above; instead every claim is recorded
|
||||
# UNCONDITIONALLY (never under needs_kv_cache_zeroing) so the worker
|
||||
# clears the rows' tag words before the new owner's first step. Only
|
||||
# CircularBufferManager appends here.
|
||||
self.new_ring_block_ids: list[int] = []
|
||||
|
||||
# Mapping from request ID to blocks to track the blocks allocated
|
||||
# for each request, so that we can free the blocks when the request
|
||||
@@ -422,6 +429,17 @@ class SingleTypeKVCacheManager(ABC):
|
||||
self.new_block_ids = []
|
||||
return ids
|
||||
|
||||
def take_new_ring_block_ids(self) -> list[int]:
|
||||
"""Drain ring blocks claimed since the last call; empty for
|
||||
every manager but CircularBufferManager."""
|
||||
ids = self.new_ring_block_ids
|
||||
self.new_ring_block_ids = []
|
||||
return ids
|
||||
|
||||
def get_ring_block_ids(self, request_id: str) -> list[int]:
|
||||
"""The request's ring block ids; empty for non-ring managers."""
|
||||
return []
|
||||
|
||||
def take_pending_cow_copies(
|
||||
self,
|
||||
) -> list[tuple[KVCacheBlock, KVCacheBlock]]:
|
||||
@@ -591,6 +609,26 @@ class SingleTypeKVCacheManager(ABC):
|
||||
# Free blocks in reverse order so that the tail blocks are freed first.
|
||||
self.block_pool.free_blocks(reversed(self.pop_blocks_for_free(request_id)))
|
||||
|
||||
def pop_retained_checkpoints(self, request_id: str) -> list[KVCacheBlock]:
|
||||
"""Pop the prefix-cache checkpoints held for ``request_id`` outside its
|
||||
block table, without freeing them. Only align-mode Mamba holds any
|
||||
(see ``MambaManager._retire_state_blocks``). The coordinator frees them
|
||||
after every group's table, so they are the request's last blocks to
|
||||
become eviction candidates."""
|
||||
return []
|
||||
|
||||
def get_num_retained_checkpoint_blocks_to_reserve(
|
||||
self,
|
||||
request_id: str,
|
||||
num_tokens: int,
|
||||
new_computed_blocks: Sequence[KVCacheBlock] = (),
|
||||
num_local_computed_tokens: int = 0,
|
||||
) -> int:
|
||||
"""Blocks the admission gate must add for checkpoints this request
|
||||
will hold outside its block table (0 for every manager but align-mode
|
||||
Mamba with retention on)."""
|
||||
return 0
|
||||
|
||||
@abstractmethod
|
||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
|
||||
"""Get the number of common prefix blocks for all requests with allocated
|
||||
@@ -1199,23 +1237,160 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
return 0
|
||||
|
||||
|
||||
class _RingSlotPage:
|
||||
"""One pool block shared by up to ``slots`` requests' rings."""
|
||||
|
||||
__slots__ = ("block", "free_slots", "live")
|
||||
|
||||
def __init__(self, block: KVCacheBlock, slots: int) -> None:
|
||||
self.block = block
|
||||
# Highest first, so pop() hands out the lowest free slot.
|
||||
self.free_slots = list(range(slots - 1, -1, -1))
|
||||
self.live = 0
|
||||
|
||||
|
||||
class RingSlotPool:
|
||||
"""v2.5 side-cache layout: ring blocks as slots of shared pool blocks.
|
||||
|
||||
The QSA ring needs one block per request for the request's lifetime, but
|
||||
its pages for every layer of a stage are ~16 KB against a 12.9 MB pool
|
||||
block. This pool takes whole blocks from the main ``BlockPool`` and cuts
|
||||
each into ``slots_per_block`` slots; a slot's block id is
|
||||
``pool_block_id * slots_per_block + slot``, which is exactly the index of
|
||||
the slot in the worker's ring view (``KVCacheTensor.slots_per_block``).
|
||||
|
||||
The slot blocks it hands out carry ``pool=self``, so every existing free
|
||||
path -- ``SingleTypeKVCacheManager.free``, the coordinator's deferred
|
||||
``pop_blocks_for_free`` and the scheduler's fenced drain, which all call
|
||||
``BlockPool.free_blocks`` -- routes them back here, at the time the old
|
||||
per-request block would have gone back (the deferred free is kept). A pool
|
||||
block returns to the main pool when its last slot is freed. Rings are never
|
||||
prefix-cached, so a slot block carries no hash.
|
||||
"""
|
||||
|
||||
def __init__(self, block_pool: BlockPool, slots_per_block: int) -> None:
|
||||
assert slots_per_block > 1
|
||||
self.block_pool = block_pool
|
||||
self.slots_per_block = slots_per_block
|
||||
# Pool block id -> page, in claim order (the oldest page fills first).
|
||||
self._pages: dict[int, _RingSlotPage] = {}
|
||||
|
||||
@property
|
||||
def num_pages(self) -> int:
|
||||
"""Pool blocks currently held for rings."""
|
||||
return len(self._pages)
|
||||
|
||||
@property
|
||||
def num_live_slots(self) -> int:
|
||||
return sum(page.live for page in self._pages.values())
|
||||
|
||||
def has_free_slot(self) -> bool:
|
||||
return any(page.free_slots for page in self._pages.values())
|
||||
|
||||
def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]:
|
||||
out: list[KVCacheBlock] = []
|
||||
for _ in range(num_blocks):
|
||||
page = next((p for p in self._pages.values() if p.free_slots), None)
|
||||
if page is None:
|
||||
(pool_block,) = self.block_pool.get_new_blocks(1)
|
||||
page = _RingSlotPage(pool_block, self.slots_per_block)
|
||||
self._pages[pool_block.block_id] = page
|
||||
slot = page.free_slots.pop()
|
||||
page.live += 1
|
||||
out.append(
|
||||
KVCacheBlock(
|
||||
page.block.block_id * self.slots_per_block + slot,
|
||||
ref_cnt=1,
|
||||
pool=self, # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
def free_blocks(self, ordered_blocks) -> None:
|
||||
released: list[KVCacheBlock] = []
|
||||
for block in ordered_blocks:
|
||||
assert block.pool is self, "not a ring slot of this pool"
|
||||
block.ref_cnt -= 1
|
||||
assert block.ref_cnt == 0, "ring slots have a single owner"
|
||||
page_id, slot = divmod(block.block_id, self.slots_per_block)
|
||||
page = self._pages[page_id]
|
||||
assert slot not in page.free_slots, f"slot {block.block_id} freed twice"
|
||||
page.free_slots.append(slot)
|
||||
page.free_slots.sort(reverse=True)
|
||||
page.live -= 1
|
||||
if page.live == 0:
|
||||
del self._pages[page_id]
|
||||
released.append(page.block)
|
||||
if released:
|
||||
self.block_pool.free_blocks(released)
|
||||
|
||||
|
||||
class CircularBufferManager(FullAttentionManager):
|
||||
"""Claims the ring's single block per request; prefix caching disabled."""
|
||||
|
||||
supports_fine_grained_hash_lookup: ClassVar[bool] = False
|
||||
|
||||
# v2.5 side-cache layout: set by the coordinator for a slot-packed group.
|
||||
_slot_pool: RingSlotPool | None = None
|
||||
|
||||
def enable_slot_packing(self, slots_per_block: int) -> None:
|
||||
"""Claim ring slots of shared pool blocks instead of whole blocks."""
|
||||
assert not self.req_to_blocks, "enable slot packing before any claim"
|
||||
self._slot_pool = RingSlotPool(self.block_pool, slots_per_block)
|
||||
|
||||
@property
|
||||
def slot_pool(self) -> RingSlotPool | None:
|
||||
return self._slot_pool
|
||||
|
||||
if self._slot_pool is not None:
|
||||
new_blocks = self._slot_pool.get_new_blocks(1)
|
||||
else:
|
||||
new_blocks = self.block_pool.get_new_blocks(1)
|
||||
|
||||
def _claim_ring_block(
|
||||
self, request_id: str, record_for_zeroing: bool = True
|
||||
) -> list[KVCacheBlock]:
|
||||
req_blocks = self.req_to_blocks[request_id]
|
||||
if req_blocks:
|
||||
return []
|
||||
new_blocks = self.block_pool.get_new_blocks(1)
|
||||
if self._slot_pool is not None:
|
||||
new_blocks = self._slot_pool.get_new_blocks(1)
|
||||
else:
|
||||
new_blocks = self.block_pool.get_new_blocks(1)
|
||||
req_blocks.extend(new_blocks)
|
||||
if self._record_new_block_ids and record_for_zeroing:
|
||||
self.new_block_ids.extend(block.block_id for block in new_blocks)
|
||||
# Unconditional -- not gated on needs_kv_cache_zeroing.
|
||||
self.new_ring_block_ids.extend(block.block_id for block in new_blocks)
|
||||
return new_blocks
|
||||
|
||||
def take_new_ring_block_ids(self) -> list[int]:
|
||||
"""Drain the ring blocks claimed since the last call, keeping only
|
||||
blocks a request of this manager still holds, in claim order, once.
|
||||
|
||||
A block claimed and then freed within the same scheduling pass (the
|
||||
request was preempted after its allocation) may already belong to
|
||||
another group by the time the worker acts; clearing its tag words
|
||||
there would write into that group's page, so it is dropped here.
|
||||
"""
|
||||
claimed = self.new_ring_block_ids
|
||||
self.new_ring_block_ids = []
|
||||
if not claimed:
|
||||
return []
|
||||
held = {
|
||||
block.block_id for blocks in self.req_to_blocks.values() for block in blocks
|
||||
}
|
||||
out: list[int] = []
|
||||
seen: set[int] = set()
|
||||
for block_id in claimed:
|
||||
if block_id in held and block_id not in seen:
|
||||
seen.add(block_id)
|
||||
out.append(block_id)
|
||||
return out
|
||||
|
||||
def get_ring_block_ids(self, request_id: str) -> list[int]:
|
||||
return [block.block_id for block in self.req_to_blocks.get(request_id, ())]
|
||||
|
||||
def get_num_blocks_to_allocate(
|
||||
self,
|
||||
request_id: str,
|
||||
@@ -1227,7 +1402,12 @@ class CircularBufferManager(FullAttentionManager):
|
||||
apply_admission_cap: bool = False,
|
||||
prefill_end: int = 0,
|
||||
) -> int:
|
||||
return 0 if self.req_to_blocks.get(request_id) else 1
|
||||
if self.req_to_blocks.get(request_id):
|
||||
return 0
|
||||
# A free slot in a page the ring already holds costs no pool block.
|
||||
if self._slot_pool is not None and self._slot_pool.has_free_slot():
|
||||
return 0
|
||||
return 1
|
||||
|
||||
def allocate_new_blocks(
|
||||
self, request_id: str, num_tokens: int, num_tokens_main_model: int
|
||||
@@ -1489,6 +1669,191 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
# connector; a request that finishes first hands off this table
|
||||
# source directly.
|
||||
self._producer_partial_tail_reqs: dict[str, tuple[KVCacheBlock, int]] = {}
|
||||
# v2.5. In align mode a state block leaves the block table (it is
|
||||
# nulled and released) as soon as the request computes one block past
|
||||
# it -- `_remove_blocks_in_range` / `remove_skipped_blocks`. When that
|
||||
# block carries a prefix-cache hash it is the request's GDN checkpoint
|
||||
# (the replay boundary under retention interval 0), and releasing it
|
||||
# mid-request puts it in the free queue AHEAD of the request's own
|
||||
# attention blocks, which stay referenced until finish. Concurrent
|
||||
# prefills then pop it first; since the hybrid hit is the shortest
|
||||
# group's, that single block zeroes the next turn's reuse. With
|
||||
# retention on, the newest hashed retired state stays referenced here
|
||||
# (at most one per request per group) and is released only when the
|
||||
# request is freed, after its attention blocks (coordinator.free).
|
||||
self._retain_checkpoint: bool = (
|
||||
self.mamba_cache_mode == "align"
|
||||
and self.enable_caching
|
||||
and envs.VLLM_E2_MAMBA_RETAIN_CHECKPOINT
|
||||
)
|
||||
# {request_id: (hashed token position, block)}: the newest hashed
|
||||
# state the request resumed from or produced, held with one ref of
|
||||
# its own (ordered by ``block_hash_num_tokens`` rather than
|
||||
# block index so a sub-block partial-tail key sorts correctly against
|
||||
# full-block boundary states).
|
||||
#
|
||||
# With ``--prefix-match-unit`` below the block size a request
|
||||
# that resumed at a partial tail (mid-block) never hashes a boundary
|
||||
# state: the replay boundary lies below its resume point. Its only
|
||||
# reusable GDN state is its own partial-tail checkpoint, which the
|
||||
# next chunk moves into a CoW block in no block table; the scheduler
|
||||
# drops the copy's retention refs once the copy has run, so that
|
||||
# block sat at ref 0 in the free queue for the rest of the request,
|
||||
# ahead of its attention blocks, and concurrent prefills evicted it
|
||||
# (measured: 7/61 follow-ups with cached_tokens 0 at unit 64, every
|
||||
# one after a partial-tail resume; none at unit 4096). The CoW paths
|
||||
# therefore hold their hashed endpoint here too: the moved tail for
|
||||
# a running request, and the hit source a new request copies from.
|
||||
self._retained_checkpoints: dict[str, tuple[int, KVCacheBlock]] = {}
|
||||
# Callbacks (source_block, cow_block) run after a running
|
||||
# request moved its live tail's hash to the CoW copy, for refs held
|
||||
# outside this manager (the KVCacheManager's grace and waiter pins).
|
||||
self.moved_hash_listeners: list[
|
||||
Callable[[KVCacheBlock, KVCacheBlock], None]
|
||||
] = []
|
||||
if self.mamba_cache_mode == "align" and self.enable_caching:
|
||||
# The A/B evidence line: boot logs must show which arm ran.
|
||||
logger.info_once(
|
||||
"Mamba align prefix-checkpoint retention: %s "
|
||||
"(VLLM_E2_MAMBA_RETAIN_CHECKPOINT)",
|
||||
"on" if self._retain_checkpoint else "off",
|
||||
)
|
||||
if (
|
||||
self._retain_checkpoint
|
||||
and self.block_size > self.block_pool.hash_block_size
|
||||
):
|
||||
logger.info_once(
|
||||
"Mamba align: each request holds its partial-tail CoW checkpoint "
|
||||
"until it is freed (hash unit %d, block %d)",
|
||||
self.block_pool.hash_block_size,
|
||||
self.block_size,
|
||||
)
|
||||
if (
|
||||
self.mamba_cache_mode == "align"
|
||||
and self.enable_caching
|
||||
and self.block_size > self.block_pool.hash_block_size
|
||||
):
|
||||
logger.info_once(
|
||||
"Mamba align: a prompt tail key that lands on a block end is "
|
||||
"hashed too (hash unit %d, block %d)",
|
||||
self.block_pool.hash_block_size,
|
||||
self.block_size,
|
||||
)
|
||||
|
||||
def _retire_state_blocks(
|
||||
self, request_id: str, retired: list[tuple[int, KVCacheBlock]]
|
||||
) -> None:
|
||||
"""Release align-mode state blocks the request no longer reads.
|
||||
|
||||
``retired`` holds (logical index, block) pairs already nulled out of
|
||||
the block table, in the eviction order the caller wants. With
|
||||
retention on, the newest hashed one is kept referenced as the request's
|
||||
checkpoint instead, and the checkpoint it supersedes (if any) is
|
||||
released now -- so the request never holds more than one.
|
||||
"""
|
||||
if self._retain_checkpoint:
|
||||
newest: tuple[int, KVCacheBlock] | None = None
|
||||
for _, block in retired:
|
||||
if block.block_hash is None:
|
||||
continue
|
||||
position = block.block_hash_num_tokens
|
||||
assert position is not None
|
||||
if newest is None or position > newest[0]:
|
||||
newest = (position, block)
|
||||
held = self._retained_checkpoints.get(request_id)
|
||||
if newest is not None and (held is None or newest[0] > held[0]):
|
||||
# The retired block's table ref becomes the hold's ref.
|
||||
self._retained_checkpoints[request_id] = newest
|
||||
retired = [entry for entry in retired if entry[1] is not newest[1]]
|
||||
if held is not None:
|
||||
# Superseded: it ages from here like any cached free block.
|
||||
retired.append(held)
|
||||
if retired:
|
||||
self.block_pool.free_blocks([block for _, block in retired])
|
||||
|
||||
def _hold_checkpoint(self, request_id: str, block: KVCacheBlock) -> bool:
|
||||
"""Hold ``block`` (a hashed state outside the request's table) as the
|
||||
request's prefix checkpoint, taking one extra ref for the hold.
|
||||
|
||||
Only the newest position is held: an older one is left alone (it ages
|
||||
from wherever it is), a superseded one is released here. The hold's
|
||||
ref is returned by ``pop_retained_checkpoints`` -> ``free_blocks``.
|
||||
Returns whether the block is now held.
|
||||
"""
|
||||
if not self._retain_checkpoint or block.is_null or block.block_hash is None:
|
||||
return False
|
||||
position = block.block_hash_num_tokens
|
||||
assert position is not None
|
||||
held = self._retained_checkpoints.get(request_id)
|
||||
if held is not None and (held[1] is block or held[0] >= position):
|
||||
return False
|
||||
self.block_pool.touch([block])
|
||||
self._retained_checkpoints[request_id] = (position, block)
|
||||
if held is not None:
|
||||
self.block_pool.free_blocks([held[1]])
|
||||
return True
|
||||
|
||||
def _follow_moved_hash(
|
||||
self, owner_id: str, source_block: KVCacheBlock, cow_block: KVCacheBlock
|
||||
) -> None:
|
||||
"""``owner_id`` just moved ``source_block``'s hashes to ``cow_block``
|
||||
and keeps writing ``source_block``. A request that resumed at that
|
||||
tail while it was still the owner's live block holds the source for
|
||||
those hashes; move its hold to the copy, so it pins the hashed state
|
||||
and not another request's running block.
|
||||
"""
|
||||
for request_id, (position, block) in list(self._retained_checkpoints.items()):
|
||||
if request_id != owner_id and block is source_block:
|
||||
self.block_pool.touch([cow_block])
|
||||
self._retained_checkpoints[request_id] = (position, cow_block)
|
||||
self.block_pool.free_blocks([source_block])
|
||||
for listener in self.moved_hash_listeners:
|
||||
listener(source_block, cow_block)
|
||||
|
||||
def pop_retained_checkpoints(self, request_id: str) -> list[KVCacheBlock]:
|
||||
if self.mamba_cache_mode != "align":
|
||||
return []
|
||||
held = self._retained_checkpoints.pop(request_id, None)
|
||||
return [] if held is None else [held[1]]
|
||||
|
||||
def get_num_retained_checkpoint_blocks_to_reserve(
|
||||
self,
|
||||
request_id: str,
|
||||
num_tokens: int,
|
||||
new_computed_blocks: Sequence[KVCacheBlock] = (),
|
||||
num_local_computed_tokens: int = 0,
|
||||
) -> int:
|
||||
# A request that already holds a checkpoint only ever swaps it. One
|
||||
# that does not will hold one once a hashed state retires, which needs
|
||||
# at least one state block behind the running one.
|
||||
if not self._retain_checkpoint or request_id in self._retained_checkpoints:
|
||||
return 0
|
||||
# A sub-block (partial-tail) hit holds the hit source only
|
||||
# until the request's own tail is moved into a CoW block that is held
|
||||
# in its place; at that step both are referenced, so the request's
|
||||
# lifetime peak is one block above what the hit brought (whatever the
|
||||
# prompt length: the tail CoW happens inside the first block too).
|
||||
if self._has_partial_local_hit(new_computed_blocks, num_local_computed_tokens):
|
||||
return 1
|
||||
if num_tokens <= self.block_size:
|
||||
# No state block retires behind the running one, but a prompt
|
||||
# whose tail is keyed inside the block still moves that tail into
|
||||
# a held CoW block when the hash unit is finer than the block.
|
||||
return int(self.block_pool.hash_block_size < self.block_size)
|
||||
# A prefix hit already brings the block that becomes the held
|
||||
# checkpoint -- the hit state, counted by get_num_blocks_to_allocate as
|
||||
# an evictable computed block (or already referenced elsewhere) -- and
|
||||
# the same holds for a hashed state still in the request's table.
|
||||
# Reserving another one double-counts every hit admission by one block
|
||||
# per group, which only delays the follow-up turns the fix is for.
|
||||
if any(not blk.is_null for blk in new_computed_blocks):
|
||||
return 0
|
||||
if any(
|
||||
not blk.is_null and blk.block_hash is not None
|
||||
for blk in self.req_to_blocks.get(request_id, ())
|
||||
):
|
||||
return 0
|
||||
return 1
|
||||
|
||||
@classmethod
|
||||
def find_longest_cache_hit(
|
||||
@@ -1648,15 +2013,14 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
last_block = min(last_block, len(blocks))
|
||||
if first_block >= last_block:
|
||||
return
|
||||
freed: list[KVCacheBlock] = []
|
||||
freed: list[tuple[int, KVCacheBlock]] = []
|
||||
# Mamba prefill leaves null gaps between states awaiting retirement.
|
||||
for i in range(last_block - 1, first_block - 1, -1):
|
||||
if blocks[i].is_null:
|
||||
continue
|
||||
freed.append(blocks[i])
|
||||
freed.append((i, blocks[i]))
|
||||
blocks[i] = self._null_block
|
||||
if freed:
|
||||
self.block_pool.free_blocks(freed)
|
||||
self._retire_state_blocks(request_id, freed)
|
||||
self._num_retired_blocks[request_id] = last_block
|
||||
|
||||
def remove_skipped_blocks(
|
||||
@@ -1686,8 +2050,11 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
):
|
||||
blocks = self.req_to_blocks[request_id]
|
||||
if blocks[last_state_block_idx] != self._null_block:
|
||||
self.block_pool.free_blocks([blocks[last_state_block_idx]])
|
||||
block = blocks[last_state_block_idx]
|
||||
blocks[last_state_block_idx] = self._null_block
|
||||
self._retire_state_blocks(
|
||||
request_id, [(last_state_block_idx, block)]
|
||||
)
|
||||
|
||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
|
||||
"""Cascade attention is not supported by mamba"""
|
||||
@@ -1924,6 +2291,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
# overwrites source_block.
|
||||
assert req_blocks[block_idx] is source_block
|
||||
self.block_pool.move_block_hashes(source_block, cow_block)
|
||||
self._follow_moved_hash(request_id, source_block, cow_block)
|
||||
self._pending_cow_copies.append((source_block, cow_block))
|
||||
source_block.ref_cnt += 1
|
||||
producer_tail = self._producer_partial_tail_reqs.pop(
|
||||
@@ -1947,9 +2315,20 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
# The moved entry is only filled by this step's
|
||||
# copy, so defer same-step hits on it.
|
||||
self.cached_blocks_this_step.add(cow_block.block_hash)
|
||||
# The moved tail is the request's newest
|
||||
# reusable state and lives in no block table; hold it
|
||||
# past the copy's retention so the request's next
|
||||
# turn can resume at it (see __init__).
|
||||
self._hold_checkpoint(request_id, cow_block)
|
||||
else:
|
||||
self._apply_cow(request_id, block_idx, source_block, cow_block)
|
||||
returned_blocks = [cow_block] + returned_blocks
|
||||
# The hit source keeps the hash while the
|
||||
# request advances its private copy; hold it until
|
||||
# the request's own tail supersedes it, so a request
|
||||
# that stops early (preempted, aborted, block-aligned
|
||||
# prompt) still leaves the state it resumed from.
|
||||
self._hold_checkpoint(request_id, source_block)
|
||||
req_blocks.extend(new_blocks)
|
||||
self._allocated_block_reqs.add(request_id)
|
||||
self._partial_hit_reqs.pop(request_id, None)
|
||||
@@ -2000,6 +2379,12 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
for entry in self._pending_boundary_state_offloads
|
||||
if entry[0] != request_id
|
||||
]
|
||||
# The coordinator pops the retained checkpoint first and frees it
|
||||
# after every group. A direct caller still gets it back, first in
|
||||
# allocation order, so the caller's reversed free releases it last.
|
||||
retained = self.pop_retained_checkpoints(request_id)
|
||||
if retained:
|
||||
return retained + super().pop_blocks_for_free(request_id)
|
||||
return super().pop_blocks_for_free(request_id)
|
||||
|
||||
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
|
||||
@@ -2018,6 +2403,14 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
replay_boundaries: Sequence[int],
|
||||
) -> None:
|
||||
num_cached_blocks_before = self.num_cached_block.get(request.request_id, 0)
|
||||
tail = self._block_aligned_prompt_tail(request)
|
||||
if tail:
|
||||
# With a hash unit finer than the block, the prompt's tail
|
||||
# key (where the next turn resumes; `_cache_partial_tail_block`)
|
||||
# can land exactly on a block end. It is then a full-block key,
|
||||
# not a partial one, and the replay-boundary mask alone (one block
|
||||
# below it under the eagle drop) would never hash that state.
|
||||
replay_boundaries = (*replay_boundaries, tail)
|
||||
super().cache_blocks(
|
||||
request,
|
||||
num_tokens,
|
||||
@@ -2059,6 +2452,22 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
def new_step_starts(self) -> None:
|
||||
self.cached_blocks_this_step.clear()
|
||||
|
||||
def _block_aligned_prompt_tail(self, request: Request) -> int:
|
||||
"""The prompt's tail key when it falls on a block end, else 0.
|
||||
|
||||
Only with a hash unit finer than the block (at the block unit the
|
||||
tail IS the replay boundary, which the mask already keeps). The key
|
||||
is placed exactly as `_cache_partial_tail_block` places a sub-block
|
||||
tail, eagle shift included.
|
||||
"""
|
||||
hash_block_size = self.block_pool.hash_block_size
|
||||
if self.mamba_cache_mode != "align" or self.block_size <= hash_block_size:
|
||||
return 0
|
||||
tail = request.num_prompt_tokens // hash_block_size * hash_block_size
|
||||
if self.drop_eagle_checkpoint_block:
|
||||
tail = max(tail - hash_block_size, 0)
|
||||
return tail if tail % self.block_size == 0 else 0
|
||||
|
||||
def _cache_partial_tail_block(
|
||||
self,
|
||||
request: Request,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,547 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fork startup guards that fail closed on a graph profile we never qualified.
|
||||
|
||||
These run at model construction, on every rank, for the target model and for
|
||||
the MTP drafter, before profiling or capture.
|
||||
|
||||
Compile-profile guard: this tree restores torch.compile on the
|
||||
NVIDIA Qwen4Exp path (reverting vLLM #55272) and removes Qwen4Exp from
|
||||
``DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES``. The ONE qualified profile is the
|
||||
v2.0.1 production identity:
|
||||
|
||||
* ``VLLM_USE_BREAKABLE_CUDAGRAPH`` explicitly pinned to exactly ``0``, and
|
||||
resolved OFF;
|
||||
* ``CompilationMode.VLLM_COMPILE``;
|
||||
* ``CUDAGraphMode.FULL_AND_PIECEWISE``;
|
||||
* every Qwen4Exp request-dependent custom op in ``splitting_ops``
|
||||
(:data:`REQUIRED_SPLITTING_OPS`);
|
||||
* a live ``@support_torch_compile`` submodule
|
||||
(:func:`assert_compiled_submodule`), which is the property
|
||||
``vllm/v1/worker/gpu/cudagraph_utils.py:605`` actually tests at capture.
|
||||
|
||||
Anything else -- eager, NONE + FULL, FULL_DECODE_ONLY, breakable PIECEWISE --
|
||||
fails loud. ``VLLM_E2_ALLOW_BREAKABLE=1`` is the single, logged opt-out for
|
||||
deliberate non-production boots (debug, probes); a production launch must not set it.
|
||||
|
||||
Timing limit: this runs AFTER ``VllmConfig.__post_init__`` resolved the
|
||||
profile. It stops the boot before profiling and capture; it cannot recover
|
||||
the originally requested profile if the config layer rewrote it.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
BREAKABLE_ENV = "VLLM_USE_BREAKABLE_CUDAGRAPH"
|
||||
#: Opt-out for every guard in this module. Recorded in the launch identity and
|
||||
#: PROHIBITED in production profiles; a production launch must not set it.
|
||||
ALLOW_BREAKABLE_ENV = "VLLM_E2_ALLOW_BREAKABLE"
|
||||
#: Optional expected resolved cudagraph mode, by name. When set, a mismatch is
|
||||
#: an error (in addition to the fixed requirement below).
|
||||
EXPECTED_MODE_ENV = "VLLM_E2_EXPECTED_CUDAGRAPH_MODE"
|
||||
|
||||
#: The qualified profile (the compile-profile guard).
|
||||
REQUIRED_COMPILATION_MODE_NAME = "VLLM_COMPILE"
|
||||
REQUIRED_CUDAGRAPH_MODE_NAME = "FULL_AND_PIECEWISE"
|
||||
#: The only raw VLLM_USE_BREAKABLE_CUDAGRAPH value accepted as the production
|
||||
#: pin. Resolution semantics stay upstream's (_upstream_truth).
|
||||
PRODUCTION_BREAKABLE_PIN = "0"
|
||||
|
||||
#: Custom ops that must stay graph breaks on the NVIDIA Qwen4Exp path. Each
|
||||
#: reads per-step request metadata (or forks/joins a side stream) and would be
|
||||
#: baked into a piecewise graph otherwise.
|
||||
REQUIRED_SPLITTING_OPS = (
|
||||
"vllm::qwen4_exp_qsa_with_output",
|
||||
"vllm::qwen4_exp_ple_short_conv",
|
||||
"vllm::qwen4_exp_compute_ple_ngram_ids",
|
||||
"vllm::qwen4_exp_ple_start_prefetch",
|
||||
"vllm::qwen4_exp_ple_finalize_prefetch",
|
||||
"vllm::qwen_gdn_attention_core",
|
||||
"vllm::qwen_gdn_attention_core_fused_norm_packed",
|
||||
"vllm::ple_offload_wait_copy",
|
||||
)
|
||||
|
||||
#: Upstream v0.30.0 auto-enabled breakable cudagraphs for these; this tree
|
||||
#: does not (vllm/config/vllm.py). Kept to name the mechanism in messages.
|
||||
OUR_AUTO_BREAKABLE_ARCHITECTURES = frozenset(
|
||||
{
|
||||
"Qwen4ExpForCausalLM",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
"Qwen4ExpMTP",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _upstream_truth(raw: str | None) -> bool:
|
||||
"""Upstream's exact semantics for the breakable switch.
|
||||
|
||||
``envs.py:937`` is ``os.environ.get(..., "0") == "1"``. Nothing else is on.
|
||||
"""
|
||||
return raw == "1"
|
||||
|
||||
|
||||
def breakable_resolved() -> bool:
|
||||
"""Whether breakable cudagraphs are on, as the engine will see it.
|
||||
|
||||
The raw environment and the (cached) envs accessor are two readings of one
|
||||
source; either saying "on" means on. No exception is swallowed.
|
||||
"""
|
||||
if _upstream_truth(os.environ.get(BREAKABLE_ENV)):
|
||||
return True
|
||||
import vllm.envs as envs
|
||||
|
||||
return bool(envs.VLLM_USE_BREAKABLE_CUDAGRAPH)
|
||||
|
||||
|
||||
def _name(value) -> str:
|
||||
return getattr(value, "name", str(value))
|
||||
|
||||
|
||||
def _profile(
|
||||
vllm_config, role: str = "target"
|
||||
) -> tuple[list[str], object, object, list[str] | None]:
|
||||
"""Resolved (architectures, mode, cudagraph mode, splitting ops).
|
||||
|
||||
For ``role="draft"`` the architectures are the DRAFT model's
|
||||
(``speculative_config.draft_model_config``): the drafter is constructed
|
||||
with the target's VllmConfig, whose model_config names the target. The reference host
|
||||
logged the target architecture on the draft line before this fix.
|
||||
The graph profile is the engine's single CompilationConfig for both roles.
|
||||
"""
|
||||
model_config = getattr(vllm_config, "model_config", None)
|
||||
if role == "draft":
|
||||
spec = getattr(vllm_config, "speculative_config", None)
|
||||
draft_config = getattr(spec, "draft_model_config", None)
|
||||
if draft_config is not None:
|
||||
model_config = draft_config
|
||||
if model_config is None:
|
||||
raise RuntimeError("e2-guard: vllm_config has no model_config to check")
|
||||
compilation_config = getattr(vllm_config, "compilation_config", None)
|
||||
if compilation_config is None:
|
||||
raise RuntimeError("e2-guard: vllm_config has no compilation_config to check")
|
||||
architectures = list(getattr(model_config, "architectures", None) or [])
|
||||
return (
|
||||
architectures,
|
||||
compilation_config.mode,
|
||||
compilation_config.cudagraph_mode,
|
||||
getattr(compilation_config, "splitting_ops", None),
|
||||
)
|
||||
|
||||
|
||||
def _opted_out(role: str, detail: str) -> bool:
|
||||
if os.environ.get(ALLOW_BREAKABLE_ENV, "") != "1":
|
||||
return False
|
||||
logger.warning(
|
||||
"e2-guard[%s]: %s=1 -- guards disabled by request (%s). This boot is "
|
||||
"NOT a promotion-grade boot; a production launch must not set this flag.",
|
||||
role,
|
||||
ALLOW_BREAKABLE_ENV,
|
||||
detail,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def assert_graph_profile_is_executable(vllm_config, *, role: str = "target") -> None:
|
||||
"""Require the qualified profile: VLLM_COMPILE + FULL_AND_PIECEWISE.
|
||||
|
||||
``role`` is ``"target"`` or ``"draft"``; both are checked, because
|
||||
``Qwen4ExpMTP`` builds ``Qwen4ExpMultiTokenPredictor`` rather than
|
||||
``Qwen4ExpForCausalLM``, from a config rebuilt in ``mtp.py``.
|
||||
"""
|
||||
architectures, mode, cudagraph_mode, splitting_ops = _profile(vllm_config, role)
|
||||
detail = (
|
||||
f"architectures={architectures} breakable={os.environ.get(BREAKABLE_ENV)} "
|
||||
f"compilation_mode={_name(mode)} cudagraph_mode={_name(cudagraph_mode)}"
|
||||
)
|
||||
if _opted_out(role, detail):
|
||||
return
|
||||
|
||||
# 1. The pin must be explicit, so a future upstream merge that re-adds us
|
||||
# to DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES cannot flip it silently.
|
||||
raw_breakable = os.environ.get(BREAKABLE_ENV)
|
||||
if raw_breakable is None:
|
||||
raise RuntimeError(
|
||||
f"e2-guard[{role}]: {BREAKABLE_ENV} is not set. Pin it in the "
|
||||
f"launcher: export {BREAKABLE_ENV}=0, or set {ALLOW_BREAKABLE_ENV}=1 "
|
||||
"to take an unpinned boot on purpose."
|
||||
)
|
||||
|
||||
# 2. Breakable capture is not the qualified identity.
|
||||
if raw_breakable == "1" or breakable_resolved():
|
||||
raise RuntimeError(
|
||||
f"e2-guard[{role}]: {BREAKABLE_ENV} resolved ON ({detail}). The "
|
||||
"qualified profile is breakable-off VLLM_COMPILE + "
|
||||
f"FULL_AND_PIECEWISE. Export {BREAKABLE_ENV}=0, or set "
|
||||
f"{ALLOW_BREAKABLE_ENV}=1 to take it deliberately."
|
||||
)
|
||||
|
||||
# 2b. The production pin is the exact string "0".
|
||||
# Upstream's accessor reads anything but "1" as off, so "2", "true",
|
||||
# "" or "00" would resolve off -- but a mistyped launcher value must
|
||||
# not be recorded as a qualified breakable-off profile. The logged
|
||||
# debug opt-out (VLLM_E2_ALLOW_BREAKABLE=1, handled above) is separate.
|
||||
if raw_breakable != PRODUCTION_BREAKABLE_PIN:
|
||||
raise RuntimeError(
|
||||
f"e2-guard[{role}]: {BREAKABLE_ENV}={raw_breakable!r} is not the "
|
||||
f"production pin; it must be exactly {PRODUCTION_BREAKABLE_PIN!r}. "
|
||||
"Upstream would read it as off, but the launch profile does not "
|
||||
f"say what was meant. Export {BREAKABLE_ENV}=0, or set "
|
||||
f"{ALLOW_BREAKABLE_ENV}=1 for a deliberate non-production boot."
|
||||
)
|
||||
|
||||
# 3. The profile itself: mode AND cudagraph mode, by name.
|
||||
if (
|
||||
_name(mode) != REQUIRED_COMPILATION_MODE_NAME
|
||||
or _name(cudagraph_mode) != REQUIRED_CUDAGRAPH_MODE_NAME
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"e2-guard[{role}]: resolved compilation_mode={_name(mode)} "
|
||||
f"cudagraph_mode={_name(cudagraph_mode)}; the qualified profile is "
|
||||
f"compilation_mode={REQUIRED_COMPILATION_MODE_NAME} "
|
||||
f"cudagraph_mode={REQUIRED_CUDAGRAPH_MODE_NAME} (compile-profile guard). Check "
|
||||
"-O / --compilation-config / --enforce-eager / TORCH_COMPILE_DISABLE, "
|
||||
f"or set {ALLOW_BREAKABLE_ENV}=1 for a deliberate non-production boot."
|
||||
)
|
||||
|
||||
# 4. The graph breaks that make PIECEWISE correct on this model.
|
||||
if splitting_ops is None:
|
||||
raise RuntimeError(
|
||||
f"e2-guard[{role}]: compilation_config.splitting_ops is unresolved; "
|
||||
"cannot confirm the Qwen4Exp graph breaks."
|
||||
)
|
||||
missing = [op for op in REQUIRED_SPLITTING_OPS if op not in splitting_ops]
|
||||
if missing:
|
||||
raise RuntimeError(
|
||||
f"e2-guard[{role}]: splitting_ops is missing {missing}. Each of "
|
||||
"these reads per-step request metadata (or forks/joins a side "
|
||||
"stream) and would be frozen into a piecewise CUDA graph."
|
||||
)
|
||||
|
||||
# 5. Optional exact pin of the resolved mode, for a launch profile.
|
||||
expected = os.environ.get(EXPECTED_MODE_ENV, "").strip()
|
||||
if expected and _name(cudagraph_mode) != expected:
|
||||
raise RuntimeError(
|
||||
f"e2-guard[{role}]: resolved cudagraph_mode={_name(cudagraph_mode)} "
|
||||
f"but {EXPECTED_MODE_ENV}={expected}. The launch profile and the "
|
||||
"engine disagree about the graph identity."
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"e2-guard[%s]: architectures=%s breakable=OFF compilation_mode=%s "
|
||||
"cudagraph_mode=%s splitting_ops=+%d required. Recorded, not "
|
||||
"certified -- this checks startup state, not qualification.",
|
||||
role,
|
||||
architectures,
|
||||
_name(mode),
|
||||
_name(cudagraph_mode),
|
||||
len(REQUIRED_SPLITTING_OPS),
|
||||
)
|
||||
|
||||
|
||||
def assert_compiled_submodule(module, vllm_config, *, role: str = "target") -> None:
|
||||
"""Require a live ``@support_torch_compile`` wrapper on ``module``.
|
||||
|
||||
This is the property ``has_compiled_submodule()`` checks before PIECEWISE
|
||||
capture (``gpu/cudagraph_utils.py:132``, ``:605``). Checking it at
|
||||
construction turns a mid-capture failure into a startup one.
|
||||
"""
|
||||
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
|
||||
|
||||
compiled = isinstance(module, TorchCompileWithNoGuardsWrapper) and not getattr(
|
||||
module, "do_not_compile", True
|
||||
)
|
||||
if compiled:
|
||||
return
|
||||
detail = (
|
||||
f"{type(module).__name__} is_wrapper="
|
||||
f"{isinstance(module, TorchCompileWithNoGuardsWrapper)} "
|
||||
f"do_not_compile={getattr(module, 'do_not_compile', '<absent>')}"
|
||||
)
|
||||
if _opted_out(role, detail):
|
||||
return
|
||||
raise RuntimeError(
|
||||
f"e2-guard[{role}]: {detail}: no live torch.compile wrapper, so "
|
||||
"PIECEWISE capture would raise at gpu/cudagraph_utils.py:605. The "
|
||||
"@support_torch_compile decorator (compile-profile guard) is missing or disabled."
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Resolved runtime graph modes.
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: Target architectures whose V2 capture is checked. The construction guard
|
||||
#: above lives in the NVIDIA Qwen4Exp model; this tree targets sm_86 only.
|
||||
QWEN4EXP_TARGET_ARCHITECTURES = frozenset(
|
||||
{"Qwen4ExpForCausalLM", "Qwen4ExpForConditionalGeneration"}
|
||||
)
|
||||
|
||||
#: What each V2 cudagraph manager may hold immediately before capture.
|
||||
#: The construction guard checks ``compilation_config.cudagraph_mode``; the
|
||||
#: V2 runner later calls ``resolve_cudagraph_mode_and_sizes`` (and, with
|
||||
#: adaptive verification, ``resolve_adaptive_cudagraph_mode``), which may
|
||||
#: downgrade to FULL_DECODE_ONLY / PIECEWISE / NONE or drop the piecewise half
|
||||
#: (attention CG support, spec-decode uniform batches, no compiled submodule).
|
||||
#: No such downgrade is allowed for the target. The drafter's managers are
|
||||
#: derived from the target's resolved mode by the speculator itself:
|
||||
#: * autoregressive / MTP (``AutoRegressiveSpeculator.init_cudagraph_manager``):
|
||||
#: prefill = the target's mode unchanged; decode = FULL_DECODE_ONLY
|
||||
#: ("PIECEWISE cudagraphs are not supported for draft decodes");
|
||||
#: * multi-module MTP (``MultiModuleMTPSpeculator``): one manager,
|
||||
#: FULL_AND_PIECEWISE -> FULL_DECODE_ONLY by design.
|
||||
#: Those decode-specific modes are the only drafter deviations accepted.
|
||||
EXPECTED_RESOLVED_MODES = {
|
||||
"target": REQUIRED_CUDAGRAPH_MODE_NAME,
|
||||
"draft.prefill_cudagraph_manager": REQUIRED_CUDAGRAPH_MODE_NAME,
|
||||
"draft.decode_cudagraph_manager": "FULL_DECODE_ONLY",
|
||||
"draft.cudagraph_manager": "FULL_DECODE_ONLY",
|
||||
}
|
||||
_DRAFT_MANAGER_ATTRS = (
|
||||
"prefill_cudagraph_manager",
|
||||
"decode_cudagraph_manager",
|
||||
"cudagraph_manager",
|
||||
)
|
||||
|
||||
|
||||
def assert_resolved_cudagraph_modes(
|
||||
vllm_config, target_manager, speculator=None
|
||||
) -> None:
|
||||
"""Fail loud if the V2 runner is about to capture a downgraded mode.
|
||||
|
||||
Called from ``GPUModelRunner.capture_model`` (``vllm/v1/worker/gpu``)
|
||||
before any capture, including the ``profile_only`` capture and the
|
||||
"nothing to capture" early return, so a resolver downgrade to NONE cannot
|
||||
skip it. Applies only to a Qwen4Exp target; any other model is untouched.
|
||||
``VLLM_E2_ALLOW_BREAKABLE=1`` skips it, logged, like the other graph
|
||||
guards.
|
||||
"""
|
||||
model_config = getattr(vllm_config, "model_config", None)
|
||||
architectures = list(getattr(model_config, "architectures", None) or [])
|
||||
if not QWEN4EXP_TARGET_ARCHITECTURES & set(architectures):
|
||||
return
|
||||
|
||||
observed: dict[str, str] = {}
|
||||
if target_manager is None:
|
||||
observed["target"] = "<no manager>"
|
||||
else:
|
||||
observed["target"] = _name(getattr(target_manager, "cudagraph_mode", None))
|
||||
if speculator is not None:
|
||||
found = False
|
||||
for attr in _DRAFT_MANAGER_ATTRS:
|
||||
if not hasattr(speculator, attr):
|
||||
continue
|
||||
found = True
|
||||
manager = getattr(speculator, attr)
|
||||
observed[f"draft.{attr}"] = (
|
||||
"<no manager>"
|
||||
if manager is None
|
||||
else _name(getattr(manager, "cudagraph_mode", None))
|
||||
)
|
||||
if not found:
|
||||
observed["draft"] = f"<unrecognised speculator {type(speculator).__name__}>"
|
||||
|
||||
detail = " ".join(f"{key}={value}" for key, value in observed.items())
|
||||
if _opted_out("runtime", detail):
|
||||
return
|
||||
wrong = {
|
||||
key: value
|
||||
for key, value in observed.items()
|
||||
if EXPECTED_RESOLVED_MODES.get(key) != value
|
||||
}
|
||||
if wrong:
|
||||
expected = {key: EXPECTED_RESOLVED_MODES.get(key, "<none>") for key in wrong}
|
||||
raise RuntimeError(
|
||||
f"e2-guard[runtime]: resolved cudagraph modes {wrong} differ from "
|
||||
f"the qualified profile {expected} ({detail}). The V2 runner's "
|
||||
"resolver downgraded the graph identity after the construction "
|
||||
"guard passed (attention CG support, spec-decode uniform batch, "
|
||||
"or no compiled submodule); see the resolver's warning above. Set "
|
||||
f"{ALLOW_BREAKABLE_ENV}=1 only for a deliberate non-production boot."
|
||||
)
|
||||
logger.info(
|
||||
"e2-guard[runtime]: resolved cudagraph modes before capture: %s.", detail
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# PLE transport: carry the pull transport, UVA disabled.
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: The fork's pull-transport switch (strict "0"/"1"; registered and hashed in
|
||||
#: vllm/envs.py). Renamed from the E2 base's VLLM_PLE_CPU_OFFLOAD.
|
||||
PULL_TRANSPORT_ENV = "VLLM_E2_PLE_PULL_TRANSPORT"
|
||||
#: Upstream v0.30.0's switch for the pinned-host UVA PLE backend. Same name as
|
||||
#: the E2 base's pull switch, opposite default (upstream: "1").
|
||||
UPSTREAM_UVA_ENV = "VLLM_PLE_CPU_OFFLOAD"
|
||||
|
||||
|
||||
def pull_transport_enabled() -> bool:
|
||||
"""Whether the fork's pull transport is on. Strict; raises on bad values.
|
||||
|
||||
One source: the registered accessor ``envs.VLLM_E2_PLE_PULL_TRANSPORT``,
|
||||
whose parser (``envs._e2_strict_switch``) accepts exactly unset/"0"/"1".
|
||||
"""
|
||||
import vllm.envs as envs
|
||||
|
||||
return bool(envs.VLLM_E2_PLE_PULL_TRANSPORT)
|
||||
|
||||
|
||||
def check_upstream_uva_env_against_pull() -> None:
|
||||
"""Fail loud if the old name would turn upstream's UVA path on.
|
||||
|
||||
With the pull transport on, ``VLLM_PLE_CPU_OFFLOAD`` may be unset (the
|
||||
fork then supplies ``cpu_offload=False`` as the Engram default) or exactly
|
||||
``"0"``. ``"1"`` -- the E2-era spelling of "enable our offload" -- would
|
||||
enable upstream's pinned-host UVA backend beside ours: a double offload.
|
||||
Any other value is rejected rather than interpreted. With the pull
|
||||
transport off this function does nothing: the name is upstream's.
|
||||
"""
|
||||
if not pull_transport_enabled():
|
||||
return
|
||||
raw = os.environ.get(UPSTREAM_UVA_ENV)
|
||||
if raw is None or raw == "0":
|
||||
return
|
||||
if raw == "1":
|
||||
raise RuntimeError(
|
||||
f"e2-guard: {PULL_TRANSPORT_ENV}=1 and {UPSTREAM_UVA_ENV}=1. On "
|
||||
f"vLLM v0.30.0 {UPSTREAM_UVA_ENV} is UPSTREAM's switch for the "
|
||||
"pinned-host UVA PLE backend (default on), not the E2 pull "
|
||||
"transport, so this launch asks for both transports at once. Unset "
|
||||
f"{UPSTREAM_UVA_ENV} or export {UPSTREAM_UVA_ENV}=0 "
|
||||
"(the pull transport replaces that switch)."
|
||||
)
|
||||
raise ValueError(
|
||||
f"e2-guard: {PULL_TRANSPORT_ENV}=1 and {UPSTREAM_UVA_ENV}={raw!r}. "
|
||||
f'With the pull transport on, {UPSTREAM_UVA_ENV} must be unset or '
|
||||
'exactly "0"; any other value is refused rather than guessed.'
|
||||
)
|
||||
|
||||
|
||||
def pull_transport_cpu_offload_default(upstream_default: bool) -> bool:
|
||||
"""The Engram ``cpu_offload`` default, as this fork resolves it.
|
||||
|
||||
Pull off: ``upstream_default`` unchanged (the tree behaves as upstream).
|
||||
Pull on: the old-name check above, then ``False`` -- upstream's UVA
|
||||
backend is never the default beside the pull transport.
|
||||
"""
|
||||
if not pull_transport_enabled():
|
||||
return upstream_default
|
||||
check_upstream_uva_env_against_pull()
|
||||
return False
|
||||
|
||||
|
||||
def verify_engram_against_pull(engram_config) -> None:
|
||||
"""Refuse an explicit ``cpu_offload=True`` while the pull transport is on.
|
||||
|
||||
``--engram-config '{"cpu_offload": true}'`` takes precedence over every
|
||||
default, so it is checked on the resolved object, not the environment.
|
||||
"""
|
||||
if not pull_transport_enabled():
|
||||
return
|
||||
check_upstream_uva_env_against_pull()
|
||||
if engram_config is not None and bool(engram_config.cpu_offload):
|
||||
raise RuntimeError(
|
||||
f"e2-guard: {PULL_TRANSPORT_ENV}=1 but the resolved EngramConfig "
|
||||
"has cpu_offload=True (upstream's pinned-host UVA PLE backend). "
|
||||
"Both transports would own the PLE table. Drop cpu_offload from "
|
||||
"--engram-config (or set it false), or turn the pull transport off."
|
||||
)
|
||||
|
||||
|
||||
#: Effective PLE modes, as the guard names them.
|
||||
PLE_MODE_PULL = "pull" # E2 pull transport (VLLM_E2_PLE_PULL_TRANSPORT=1)
|
||||
PLE_MODE_UVA = "uva" # upstream Qwen4ExpPLEPinnedHostEmbedding
|
||||
PLE_MODE_DEVICE = "device" # upstream Qwen4ExpPLEDeviceEmbedding
|
||||
PLE_MODE_NONE = "none" # no Engram config resolved (no PLE layers / not CUDA)
|
||||
|
||||
|
||||
def resolve_ple_mode(vllm_config) -> str:
|
||||
"""The PLE transport this process will build, or an error if mixed.
|
||||
|
||||
Reads the two inputs that decide it -- the pull switch (strict) and the
|
||||
RESOLVED ``engram_config.cpu_offload`` -- and nothing else. Mixed states
|
||||
raise; there is no fallback from one transport to another.
|
||||
"""
|
||||
pull = pull_transport_enabled()
|
||||
check_upstream_uva_env_against_pull()
|
||||
engram_config = getattr(vllm_config, "engram_config", None)
|
||||
cpu_offload = (
|
||||
None if engram_config is None else bool(engram_config.cpu_offload)
|
||||
)
|
||||
if pull:
|
||||
if cpu_offload:
|
||||
raise RuntimeError(
|
||||
f"e2-guard: {PULL_TRANSPORT_ENV}=1 with a resolved Engram "
|
||||
"cpu_offload=True: the pull transport and upstream's UVA "
|
||||
"backend would both own the PLE table."
|
||||
)
|
||||
return PLE_MODE_PULL
|
||||
if cpu_offload is None:
|
||||
return PLE_MODE_NONE
|
||||
return PLE_MODE_UVA if cpu_offload else PLE_MODE_DEVICE
|
||||
|
||||
|
||||
def _role_ple_layer_ids(vllm_config, role: str) -> list[int]:
|
||||
"""The 1-based PLE layer ids the role's own decoder layers carry.
|
||||
|
||||
``role="target"``: ``model_config.hf_text_config.ple_layer_ids``.
|
||||
``role="draft"``: the drafter is constructed with the target's VllmConfig
|
||||
(the trap an earlier commit fixed for the architectures), so the target's
|
||||
model_config names the TARGET's PLE layers. The drafter's decoder layers
|
||||
are ``num_hidden_layers + i`` for ``i < mtp_num_hidden_layers``
|
||||
(``Qwen4ExpMultiTokenPredictor``), and a layer builds a PLE block only if
|
||||
``layer_idx + 1`` is in ``ple_layer_ids`` (``Qwen4ExpDecoderLayer``). The
|
||||
draft line reports exactly that intersection, read from
|
||||
``speculative_config.draft_model_config`` -- empty for this checkpoint,
|
||||
whose ids must name backbone layers (``Qwen4ExpTextConfig``).
|
||||
"""
|
||||
model_config = getattr(vllm_config, "model_config", None)
|
||||
if role == "draft":
|
||||
spec = getattr(vllm_config, "speculative_config", None)
|
||||
draft_config = getattr(spec, "draft_model_config", None)
|
||||
if draft_config is not None:
|
||||
model_config = draft_config
|
||||
text_config = getattr(model_config, "hf_text_config", None)
|
||||
ids = list(getattr(text_config, "ple_layer_ids", None) or [])
|
||||
if role != "draft":
|
||||
return ids
|
||||
num_hidden_layers = getattr(text_config, "num_hidden_layers", None)
|
||||
if num_hidden_layers is None:
|
||||
return ids
|
||||
num_mtp = int(getattr(text_config, "mtp_num_hidden_layers", 1))
|
||||
own = range(int(num_hidden_layers) + 1, int(num_hidden_layers) + num_mtp + 1)
|
||||
return [layer_id for layer_id in ids if layer_id in own]
|
||||
|
||||
|
||||
def assert_ple_mode(vllm_config, *, role: str = "target") -> str:
|
||||
"""Startup guard: resolve, assert and log the effective PLE mode.
|
||||
|
||||
Runs beside ``assert_graph_profile_is_executable`` at model construction
|
||||
for ``role="target"`` and ``role="draft"`` (the MTP drafter carries the
|
||||
target's EngramConfig and the same process environment, so a mixed
|
||||
configuration fails there too). NOT disabled by VLLM_E2_ALLOW_BREAKABLE:
|
||||
that opt-out is about graph identity, not about owning the PLE table twice.
|
||||
The logged ``ple_layer_ids`` are the role's own (:func:`_role_ple_layer_ids`).
|
||||
"""
|
||||
mode = resolve_ple_mode(vllm_config)
|
||||
ple_layer_ids = _role_ple_layer_ids(vllm_config, role)
|
||||
engram_config = getattr(vllm_config, "engram_config", None)
|
||||
logger.info(
|
||||
"e2-guard[%s]: ple_mode=%s pull_transport=%d engram_cpu_offload=%s "
|
||||
"%s=%s ple_layer_ids=%s.",
|
||||
role,
|
||||
mode,
|
||||
1 if mode == PLE_MODE_PULL else 0,
|
||||
None if engram_config is None else bool(engram_config.cpu_offload),
|
||||
UPSTREAM_UVA_ENV,
|
||||
os.environ.get(UPSTREAM_UVA_ENV, "<unset>"),
|
||||
ple_layer_ids,
|
||||
)
|
||||
return mode
|
||||
|
||||
|
||||
#: Back-compatible name for the earlier call sites.
|
||||
assert_breakable_cudagraph_pinned = assert_graph_profile_is_executable
|
||||
+123
-6
@@ -4,12 +4,13 @@ import gc
|
||||
import os
|
||||
import queue
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Callable, Generator, Sequence
|
||||
from concurrent.futures import Future
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from contextlib import ExitStack, contextmanager, suppress
|
||||
from enum import IntEnum
|
||||
from functools import partial
|
||||
from inspect import isclass, signature
|
||||
@@ -108,6 +109,103 @@ HANDSHAKE_TIMEOUT_MINS = 5
|
||||
_R = TypeVar("_R") # Return type for collective_rpc
|
||||
|
||||
|
||||
# How long EngineCore teardown may run after it deferred a late SIGTERM/SIGINT
|
||||
# before the process exits anyway. Covers the whole of engine_core.shutdown(),
|
||||
# not only the executor's worker ladder (grace + 4 s): a hang in later teardown
|
||||
# (scheduler, distributed cleanup) must not make the process unkillable.
|
||||
_TEARDOWN_SIGNAL_BOUND_S = 30.0
|
||||
|
||||
|
||||
# How long the guard's forced-exit paths wait for their diagnostic line. The
|
||||
# line is written from a helper thread; a blocked handler, a held logging lock
|
||||
# or a full stderr pipe costs at most this, never the exit.
|
||||
_TEARDOWN_EXIT_LOG_BOUND_S = 0.5
|
||||
|
||||
|
||||
def _log_bounded(emit: Callable[[], None], bound_s: float) -> None:
|
||||
"""Run ``emit`` (logging, flushes) on a daemon thread; wait <= bound_s."""
|
||||
try:
|
||||
thread = threading.Thread(
|
||||
target=emit, daemon=True, name="EngineCoreTeardownLog"
|
||||
)
|
||||
thread.start()
|
||||
thread.join(bound_s)
|
||||
except BaseException: # noqa: BLE001 -- the caller's exit must still run
|
||||
pass
|
||||
|
||||
|
||||
class _TeardownSignalGuard:
|
||||
"""EngineCore teardown: defer ONE late SIGTERM/SIGINT, bounded.
|
||||
|
||||
Dying on the first late signal would SIGKILL every worker
|
||||
(PR_SET_PDEATHSIG) before its shutdown drain; see the finally block of
|
||||
EngineCoreProc.run_engine_core. The deferral is bounded two ways:
|
||||
- a SECOND late signal restores the default disposition and re-delivers
|
||||
itself, so the process dies of it (the maintainer's escalation);
|
||||
- after the first deferred signal a watchdog ends the process with
|
||||
exit code 1 if teardown is still running _TEARDOWN_SIGNAL_BOUND_S later.
|
||||
"""
|
||||
|
||||
def __init__(self, bound_s: float | None = None) -> None:
|
||||
self.bound_s = _TEARDOWN_SIGNAL_BOUND_S if bound_s is None else bound_s
|
||||
self.deferred = 0
|
||||
self._done = threading.Event()
|
||||
self._watchdog: threading.Thread | None = None
|
||||
|
||||
def __call__(self, signum, frame) -> None:
|
||||
name = signal.Signals(signum).name
|
||||
if self._done.is_set():
|
||||
return
|
||||
self.deferred += 1
|
||||
if self.deferred == 1:
|
||||
logger.warning(
|
||||
"[shutdown] EngineCore: %s during resource teardown deferred; "
|
||||
"workers finish their shutdown (FINAL DRAIN) first (a second "
|
||||
"signal, or %.0f s of teardown, ends the process)",
|
||||
name,
|
||||
self.bound_s,
|
||||
)
|
||||
self._watchdog = threading.Thread(
|
||||
target=self._expire,
|
||||
daemon=True,
|
||||
name="EngineCoreTeardownWatchdog",
|
||||
)
|
||||
self._watchdog.start()
|
||||
return
|
||||
# Default disposition first, so a third signal also ends the process
|
||||
# while the (bounded) diagnostic line is written.
|
||||
signal.signal(signum, signal.SIG_DFL)
|
||||
_log_bounded(
|
||||
lambda: logger.warning(
|
||||
"[shutdown] EngineCore: second late signal (%s) during "
|
||||
"resource teardown; not deferring again",
|
||||
name,
|
||||
),
|
||||
_TEARDOWN_EXIT_LOG_BOUND_S,
|
||||
)
|
||||
os.kill(os.getpid(), signum)
|
||||
|
||||
def _expire(self) -> None:
|
||||
if self._done.wait(self.bound_s):
|
||||
return
|
||||
|
||||
def emit() -> None:
|
||||
logger.error(
|
||||
"[shutdown] EngineCore: resource teardown still running %.0f s "
|
||||
"after a deferred signal; exiting (code 1)",
|
||||
self.bound_s,
|
||||
)
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
with suppress(Exception):
|
||||
stream.flush()
|
||||
|
||||
_log_bounded(emit, _TEARDOWN_EXIT_LOG_BOUND_S)
|
||||
os._exit(1)
|
||||
|
||||
def disarm(self) -> None:
|
||||
self._done.set()
|
||||
|
||||
|
||||
class EngineCore:
|
||||
"""Inner loop of vLLM's Engine."""
|
||||
|
||||
@@ -764,7 +862,9 @@ class EngineCore:
|
||||
# When draft tokens are used with structured output, validate them
|
||||
# before computing the grammar bitmask for the deferred request.
|
||||
if self.check_for_draft_tokens:
|
||||
draft_token_ids = self.model_executor.take_draft_token_ids()
|
||||
draft_token_ids = self.model_executor.take_draft_token_ids(
|
||||
step_id=deferred_scheduler_output.scheduler_step
|
||||
)
|
||||
if draft_token_ids is not None:
|
||||
# Update the draft token ids in the scheduler output to
|
||||
# filter out the invalid spec tokens, which will be padded
|
||||
@@ -1439,12 +1539,29 @@ class EngineCoreProc(EngineCore):
|
||||
engine_core._send_engine_dead()
|
||||
raise e
|
||||
finally:
|
||||
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
||||
signal.signal(signal.SIGINT, signal.SIG_DFL)
|
||||
# Workers carry PR_SET_PDEATHSIG=SIGKILL (fail-closed PLE teardown),
|
||||
# so if this process dies before model_executor.shutdown() has reaped
|
||||
# them, every worker is SIGKILLed mid-shutdown and its bounds-guard
|
||||
# FINAL DRAIN never runs. The API server's process manager sends a
|
||||
# second SIGTERM right after the signal that started this shutdown;
|
||||
# with SIG_DFL here it killed teardown before the executor waited for
|
||||
# the workers. Defer ONE late signal until teardown ends; a second
|
||||
# signal or _TEARDOWN_SIGNAL_BOUND_S of teardown after the deferral
|
||||
# still ends the process (_TeardownSignalGuard), as do the
|
||||
# executor's terminate/kill ladder and the process manager's timeout
|
||||
# (kill_process_tree -> SIGKILL).
|
||||
teardown_guard = _TeardownSignalGuard()
|
||||
signal.signal(signal.SIGTERM, teardown_guard)
|
||||
signal.signal(signal.SIGINT, teardown_guard)
|
||||
if signal_callback is not None:
|
||||
signal_callback.stop()
|
||||
if engine_core is not None:
|
||||
engine_core.shutdown()
|
||||
try:
|
||||
if engine_core is not None:
|
||||
engine_core.shutdown()
|
||||
finally:
|
||||
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
||||
signal.signal(signal.SIGINT, signal.SIG_DFL)
|
||||
teardown_guard.disarm()
|
||||
if clean_shutdown:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@@ -265,8 +265,13 @@ class Executor(ABC):
|
||||
def execute_dummy_batch(self) -> None:
|
||||
self.collective_rpc("execute_dummy_batch")
|
||||
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
output: list[DraftTokenIds] = self.collective_rpc("take_draft_token_ids")
|
||||
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||
# Omit step_id when unset so plugin workers with the documented
|
||||
# no-argument take_draft_token_ids() keep working.
|
||||
kwargs = {} if step_id is None else {"step_id": step_id}
|
||||
output: list[DraftTokenIds] = self.collective_rpc(
|
||||
"take_draft_token_ids", kwargs=kwargs
|
||||
)
|
||||
return output[0]
|
||||
|
||||
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
|
||||
|
||||
@@ -5,6 +5,8 @@ import os
|
||||
import pickle
|
||||
import queue
|
||||
import signal
|
||||
import stat
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
@@ -70,11 +72,192 @@ from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.executor.abstract import Executor, FailureCallback
|
||||
from vllm.v1.executor.vllm_net_devices import set_worker_net_device
|
||||
from vllm.v1.outputs import AsyncModelRunnerOutput, DraftTokenIds, ModelRunnerOutput
|
||||
from vllm.v1.ple_offload.fatal import set_fatal_notify_fd
|
||||
from vllm.v1.worker.worker_base import WorkerWrapperBase
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
_WORKER_PARENT_DEATH_SIGNAL = signal.SIGKILL
|
||||
_PR_SET_PDEATHSIG = 1
|
||||
|
||||
|
||||
def _install_worker_parent_death_signal(expected_parent_pid: int) -> None:
|
||||
"""Have Linux kill this worker if its EngineCore parent exits."""
|
||||
if not sys.platform.startswith("linux"):
|
||||
return
|
||||
|
||||
import ctypes
|
||||
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
prctl = libc.prctl
|
||||
prctl.argtypes = [
|
||||
ctypes.c_int,
|
||||
ctypes.c_ulong,
|
||||
ctypes.c_ulong,
|
||||
ctypes.c_ulong,
|
||||
ctypes.c_ulong,
|
||||
]
|
||||
prctl.restype = ctypes.c_int
|
||||
if (
|
||||
prctl(
|
||||
_PR_SET_PDEATHSIG,
|
||||
int(_WORKER_PARENT_DEATH_SIGNAL),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
!= 0
|
||||
):
|
||||
error_number = ctypes.get_errno()
|
||||
raise OSError(error_number, os.strerror(error_number))
|
||||
|
||||
# The parent can exit between spawning this process and the prctl call.
|
||||
# Once prctl is installed, a later exit is covered by the kernel.
|
||||
if os.getppid() != expected_parent_pid:
|
||||
os.kill(os.getpid(), _WORKER_PARENT_DEATH_SIGNAL)
|
||||
os._exit(1)
|
||||
|
||||
|
||||
# Grace and post-SIGTERM wait, after a worker's death, for survivors that never
|
||||
# started their shutdown (see MultiprocExecutor._ensure_worker_termination).
|
||||
_FAILED_WORKER_WAIT_S = 1.0
|
||||
# After a worker's death, the bounded grace for survivors whose shutdown has
|
||||
# STARTED (FINAL DRAIN in progress). Every FINAL DRAIN measured on the reference host took
|
||||
# < 0.1 s (drain-path tests: the drain line lands within ~0.05 s of the
|
||||
# shutdown request); 2 s is 20x that, so a draining rank is not cut
|
||||
# (a shorter cap did), while one blocked inside its shutdown on the dead rank is ended.
|
||||
_FAILED_DRAIN_GRACE_S = 2.0
|
||||
# Per-worker shared byte (WorkerProcHandle.shutdown_state): written by the
|
||||
# worker in worker_main's finally, read by the executor's termination ladder.
|
||||
_SHUTDOWN_NOT_STARTED = 0
|
||||
_SHUTDOWN_STARTED = 1
|
||||
_SHUTDOWN_DONE = 2
|
||||
|
||||
|
||||
class WorkerShutdownRequested(RuntimeError):
|
||||
"""The busy loop's dequeue was cancelled by a shutdown request.
|
||||
|
||||
Raised only by WorkerProc.worker_busy_loop when the RPC broadcast queue
|
||||
reports its own cancellation (shm_broadcast: RuntimeError("cancelled")
|
||||
after MessageQueue.shutdown(), which the death-pipe monitor calls once the
|
||||
parent closes the pipe). worker_main takes the orderly exit path on this
|
||||
type only; any other exception is a failure, even one that races a
|
||||
shutdown request. Subclasses RuntimeError so existing catchers still match.
|
||||
"""
|
||||
|
||||
|
||||
def _exit_worker_process(exit_code: int) -> None:
|
||||
"""End a worker process whose orderly shutdown has already run.
|
||||
|
||||
Called after WorkerProc.shutdown(destroy_distributed=False): the FINAL
|
||||
DRAIN, the instrumentation flushes and the PLE offload / shared-memory
|
||||
cleanup are done. What would remain is process-exit-only teardown --
|
||||
destroying the NCCL communicators (PyNcclCommunicator.destroy joins its
|
||||
ncclCommAbort thread for up to 5 s per communicator while CUDA graphs that
|
||||
captured NCCL ops are still alive) and interpreter finalization (CUDA
|
||||
context, allocator pools). The driver and kernel reclaim all of it at
|
||||
process exit, and without this the executor's 5 s grace expired and the
|
||||
worker was SIGKILLed at ~9 s anyway. Flush the logs,
|
||||
then leave.
|
||||
"""
|
||||
logger.info(
|
||||
"[shutdown] WorkerProc: shutdown complete; exiting process (code %d) "
|
||||
"without distributed teardown",
|
||||
exit_code,
|
||||
)
|
||||
try:
|
||||
import logging
|
||||
|
||||
logging.shutdown()
|
||||
except Exception: # noqa: BLE001 -- the exit must happen regardless
|
||||
pass
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
with suppress(Exception):
|
||||
stream.flush()
|
||||
os._exit(exit_code)
|
||||
|
||||
|
||||
def _worker_exit_waitables(
|
||||
procs: Sequence[BaseProcess],
|
||||
) -> list[tuple[Any, BaseProcess, int | None]]:
|
||||
"""What the worker monitor waits on to learn that a worker process exited.
|
||||
|
||||
A pidfd (Linux >= 5.3) where possible: it becomes readable when the process
|
||||
itself exits. proc.sentinel is only the read end of a pipe whose write end
|
||||
the worker holds, so it reports the exit of the LAST process holding that
|
||||
write end -- and anything the worker fork+execs inherits it. Every PLE-
|
||||
owning worker does: registering its host buffers under the "file_system"
|
||||
sharing strategy starts a torch_shm_manager child, which keeps the write
|
||||
end, and which lives while any client -- rank 0's PLE child, for a TP1
|
||||
worker's buffers -- still maps them. In a rank-1 fault test PP0_TP1 died
|
||||
in release_outputs, sat as a zombie, its sentinel never fired and the
|
||||
engine hung; rank 0's death only looked detected because its PLE child
|
||||
(and so both managers) died with it, ~8 s later.
|
||||
|
||||
Returns (waitable, proc, pidfd-to-close-or-None) per process, falling back
|
||||
to the sentinel where no pidfd can be opened.
|
||||
"""
|
||||
pidfd_open = getattr(os, "pidfd_open", None)
|
||||
waitables: list[tuple[Any, BaseProcess, int | None]] = []
|
||||
try:
|
||||
for proc in procs:
|
||||
pidfd: int | None = None
|
||||
pid = getattr(proc, "pid", None)
|
||||
if pidfd_open is not None and pid is not None:
|
||||
try:
|
||||
pidfd = pidfd_open(pid)
|
||||
except OSError:
|
||||
# ESRCH (already reaped) or no pidfd support: the sentinel.
|
||||
pidfd = None
|
||||
try:
|
||||
waitable = pidfd if pidfd is not None else proc.sentinel
|
||||
waitables.append((waitable, proc, pidfd))
|
||||
except BaseException:
|
||||
# Not yet in `waitables`: the outer handler would miss it.
|
||||
if pidfd is not None:
|
||||
os.close(pidfd)
|
||||
raise
|
||||
except BaseException:
|
||||
# Nothing is handed out on failure: close what was opened.
|
||||
for _, _, opened in waitables:
|
||||
if opened is not None:
|
||||
with suppress(OSError):
|
||||
os.close(opened)
|
||||
raise
|
||||
return waitables
|
||||
|
||||
|
||||
def _make_inherited_pipes_close_on_exec() -> None:
|
||||
"""Keep this worker's inherited pipe fds out of the programs it execs.
|
||||
|
||||
Under the spawn start method multiprocessing hands the worker its end of
|
||||
the sentinel pipe (and the ready / death pipes) as INHERITABLE fds, so any
|
||||
program the worker fork+execs keeps them open: the torch_shm_manager that
|
||||
the PLE host-buffer registration starts did, and outlived the worker, so
|
||||
neither the sentinel nor the ready pipe reported its death (rank-1 fault injection).
|
||||
Nothing this process execs needs them -- multiprocessing children get
|
||||
their fds passed explicitly, subprocess closes them by default -- so mark
|
||||
every inherited pipe above stderr close-on-exec. The executor's pidfd
|
||||
wait is the primary detection; this only stops the leak at its source.
|
||||
"""
|
||||
if not sys.platform.startswith("linux"):
|
||||
return
|
||||
try:
|
||||
fds = [int(name) for name in os.listdir("/proc/self/fd")]
|
||||
except (OSError, ValueError):
|
||||
return
|
||||
for fd in fds:
|
||||
if fd <= 2:
|
||||
continue
|
||||
try:
|
||||
if stat.S_ISFIFO(os.fstat(fd).st_mode) and os.get_inheritable(fd):
|
||||
os.set_inheritable(fd, False)
|
||||
except OSError:
|
||||
# The listing's own directory fd, or one closed since.
|
||||
continue
|
||||
|
||||
|
||||
class FutureWrapper(Future):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -206,6 +389,7 @@ class MultiprocExecutor(Executor):
|
||||
if inherited_fds is not None:
|
||||
inherited_fds.append(unready_worker_handle.death_writer.fileno())
|
||||
inherited_fds.append(unready_worker_handle.ready_pipe.fileno())
|
||||
inherited_fds.append(unready_worker_handle.fault_reader.fileno())
|
||||
|
||||
# Workers must be created before wait_for_ready to avoid
|
||||
# deadlock, since worker.init_device() does a device sync.
|
||||
@@ -302,21 +486,78 @@ class MultiprocExecutor(Executor):
|
||||
# Monitors worker process liveness. If any die unexpectedly,
|
||||
# logs an error, shuts down the executor and invokes the failure
|
||||
# callback to inform the engine.
|
||||
#
|
||||
# Two wake sources per worker:
|
||||
# - its exit: a pidfd, readable when the process itself has exited
|
||||
# (fallback: proc.sentinel, which a fork+exec'd child holding the
|
||||
# write end -- torch_shm_manager from the PLE registration -- keeps
|
||||
# silent; see _worker_exit_waitables);
|
||||
# - its fault pipe: one byte from the fail-closed path right before
|
||||
# os._exit, so the failure ladder starts -- and ends the peers --
|
||||
# without waiting for the process teardown, which can itself wait
|
||||
# on a live peer. A worker closing its end on the way out (EOF, no
|
||||
# byte) is not a fault; its exit follows.
|
||||
# In a rank-1 fault test a fault on PP0_TP1 was never detected and the
|
||||
# engine hung on the response queue for 300 s.
|
||||
# Opened here, before the thread starts, so a worker that dies early
|
||||
# is still a zombie (pidfds work on zombies) rather than reaped.
|
||||
exit_waitables = _worker_exit_waitables([h.proc for h in workers])
|
||||
|
||||
def monitor_workers():
|
||||
sentinels = [h.proc.sentinel for h in workers]
|
||||
died = multiprocessing.connection.wait(sentinels)
|
||||
exit_to_handle = {
|
||||
w: h for (w, _, _), h in zip(exit_waitables, workers)
|
||||
}
|
||||
fault_to_handle = {
|
||||
h.fault_reader: h
|
||||
for h in workers
|
||||
if getattr(h, "fault_reader", None) is not None
|
||||
}
|
||||
waiting: list[Any] = list(exit_to_handle) + list(fault_to_handle)
|
||||
handle = None
|
||||
reported_fatal = False
|
||||
try:
|
||||
while handle is None:
|
||||
ready = multiprocessing.connection.wait(waiting)
|
||||
for obj in ready:
|
||||
if obj in exit_to_handle:
|
||||
handle = exit_to_handle[obj]
|
||||
break
|
||||
try:
|
||||
data = os.read(obj.fileno(), 1)
|
||||
except OSError:
|
||||
data = b""
|
||||
if data:
|
||||
handle = fault_to_handle[obj]
|
||||
reported_fatal = True
|
||||
break
|
||||
waiting.remove(obj)
|
||||
finally:
|
||||
for _, _, pidfd in exit_waitables:
|
||||
if pidfd is not None:
|
||||
with suppress(OSError):
|
||||
os.close(pidfd)
|
||||
_self = self_ref()
|
||||
if not _self or getattr(_self, "shutting_down", False):
|
||||
logger.debug("MultiprocWorkerMonitor: shutdown already initiated")
|
||||
return
|
||||
_self.is_failed = True
|
||||
proc = next(h.proc for h in workers if h.proc.sentinel == died[0])
|
||||
logger.error(
|
||||
"Worker proc %s died unexpectedly (exit code: %s), "
|
||||
"shutting down executor.",
|
||||
proc.name,
|
||||
proc.exitcode,
|
||||
)
|
||||
proc = handle.proc
|
||||
if reported_fatal:
|
||||
logger.error(
|
||||
"Worker proc %s reported a fatal fault (fail-closed) and is "
|
||||
"exiting; shutting down executor without waiting for its "
|
||||
"process teardown. executor_shutdown_monotonic=%.9f",
|
||||
proc.name,
|
||||
time.monotonic(),
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"Worker proc %s died unexpectedly (exit code: %s), "
|
||||
"shutting down executor. executor_shutdown_monotonic=%.9f",
|
||||
proc.name,
|
||||
proc.exitcode,
|
||||
time.monotonic(),
|
||||
)
|
||||
_self.shutdown()
|
||||
callback = _self.failure_callback
|
||||
if callback is not None:
|
||||
@@ -324,9 +565,16 @@ class MultiprocExecutor(Executor):
|
||||
callback()
|
||||
|
||||
if not inline:
|
||||
Thread(
|
||||
target=monitor_workers, daemon=True, name="MultiprocWorkerMonitor"
|
||||
).start()
|
||||
try:
|
||||
Thread(
|
||||
target=monitor_workers, daemon=True, name="MultiprocWorkerMonitor"
|
||||
).start()
|
||||
except BaseException:
|
||||
for _, _, pidfd in exit_waitables:
|
||||
if pidfd is not None:
|
||||
with suppress(OSError):
|
||||
os.close(pidfd)
|
||||
raise
|
||||
return
|
||||
|
||||
monitor_workers()
|
||||
@@ -366,12 +614,16 @@ class MultiprocExecutor(Executor):
|
||||
def execute_dummy_batch(self) -> None:
|
||||
self.collective_rpc("execute_dummy_batch", unique_reply_rank=self.output_rank)
|
||||
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
def take_draft_token_ids(
|
||||
self, step_id: int | None = None
|
||||
) -> DraftTokenIds | None:
|
||||
# OPTIMIZATION: Get output only from a single worker (output_rank)
|
||||
kwargs = {} if step_id is None else {"step_id": step_id}
|
||||
return self.collective_rpc(
|
||||
"take_draft_token_ids",
|
||||
unique_reply_rank=self.output_rank,
|
||||
timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
def collective_rpc( # type: ignore[override]
|
||||
@@ -450,18 +702,45 @@ class MultiprocExecutor(Executor):
|
||||
return future if non_block else future.result()
|
||||
|
||||
@staticmethod
|
||||
def _ensure_worker_termination(worker_procs: list[BaseProcess]):
|
||||
def _ensure_worker_termination(
|
||||
worker_procs: list[BaseProcess],
|
||||
*,
|
||||
after_failure: bool = False,
|
||||
shutdown_states: list[Any] | None = None,
|
||||
):
|
||||
"""Ensure that all worker processes are terminated. Assumes workers have
|
||||
received termination requests. Waits for processing, then sends
|
||||
termination and kill signals if needed."""
|
||||
termination and kill signals if needed.
|
||||
|
||||
after_failure: a worker already died. Survivors split in two. Those
|
||||
that never started WorkerProc.shutdown() are parked in a collective
|
||||
with the dead rank (NCCL / gloo, in C, where neither the grace nor a
|
||||
SIGTERM reaches them -- in a fault-injection test two ranks ran the full
|
||||
5 s + 4 s ladder to SIGKILL): they get _FAILED_WORKER_WAIT_S to start
|
||||
their shutdown, then SIGTERM, then SIGKILL after as long again. Those
|
||||
whose shutdown_states entry shows the shutdown started (FINAL DRAIN,
|
||||
flushes) get _FAILED_DRAIN_GRACE_S from the start of the ladder, then
|
||||
SIGTERM, then SIGKILL after _FAILED_WORKER_WAIT_S: a started shutdown
|
||||
can itself block on the dead rank (in a drain fault-injection test one
|
||||
survivor had started its shutdown and never logged its FINAL DRAIN --
|
||||
the drain's device synchronize waits on work paired with the dead
|
||||
rank -- and ran the full orderly ladder, workers gone 17.6 s after the
|
||||
fault). Before the drain-order fix the engine died on the late SIGTERM and
|
||||
PDEATHSIG killed all survivors at once. A worker with no state (None,
|
||||
or no shutdown_states) counts as parked. The orderly ladder (no
|
||||
failure) is unchanged.
|
||||
"""
|
||||
grace_s = float(envs.VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS)
|
||||
term_wait_s = 4.0
|
||||
|
||||
def wait_for_termination(procs, timeout):
|
||||
if not time:
|
||||
# If we are in late stage shutdown, the interpreter may replace
|
||||
# `time` with `None`.
|
||||
return all(not proc.is_alive() for proc in procs)
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
# Monotonic: a wall-clock step must not cut or stretch the grace.
|
||||
start_time = time.monotonic()
|
||||
while time.monotonic() - start_time < timeout:
|
||||
if all(not proc.is_alive() for proc in procs):
|
||||
return True
|
||||
time.sleep(0.1)
|
||||
@@ -475,9 +754,42 @@ class MultiprocExecutor(Executor):
|
||||
"[shutdown] Executor: waiting for worker exit count=%d",
|
||||
initial_count,
|
||||
)
|
||||
if wait_for_termination(
|
||||
active_procs(), timeout=envs.VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS
|
||||
):
|
||||
if after_failure and time:
|
||||
start = time.monotonic()
|
||||
states = list(shutdown_states or [None] * len(worker_procs))
|
||||
|
||||
def parked() -> list[BaseProcess]:
|
||||
return [
|
||||
proc
|
||||
for proc, state in zip(worker_procs, states)
|
||||
if proc.is_alive()
|
||||
and (state is None or state.value == _SHUTDOWN_NOT_STARTED)
|
||||
]
|
||||
|
||||
if not wait_for_termination(active_procs(), _FAILED_WORKER_WAIT_S):
|
||||
stuck = parked()
|
||||
if stuck:
|
||||
logger.warning(
|
||||
"[shutdown] Executor: workers parked after a worker "
|
||||
"failure (shutdown not started); sending SIGTERM count=%d",
|
||||
len(stuck),
|
||||
)
|
||||
for p in stuck:
|
||||
p.terminate()
|
||||
if not wait_for_termination(stuck, _FAILED_WORKER_WAIT_S):
|
||||
stuck = [p for p in stuck if p.is_alive()]
|
||||
logger.warning(
|
||||
"[shutdown] Executor: parked workers still running "
|
||||
"after SIGTERM; sending SIGKILL count=%d",
|
||||
len(stuck),
|
||||
)
|
||||
for p in stuck:
|
||||
p.kill()
|
||||
# Workers that started their shutdown get a BOUNDED grace, counted
|
||||
# from the start of this ladder, then SIGTERM, then SIGKILL.
|
||||
grace_s = max(0.0, _FAILED_DRAIN_GRACE_S - (time.monotonic() - start))
|
||||
term_wait_s = _FAILED_WORKER_WAIT_S
|
||||
if wait_for_termination(active_procs(), timeout=grace_s):
|
||||
logger.info_once("[shutdown] Executor: all workers exited gracefully")
|
||||
return
|
||||
|
||||
@@ -490,7 +802,7 @@ class MultiprocExecutor(Executor):
|
||||
)
|
||||
for p in remaining:
|
||||
p.terminate()
|
||||
if not wait_for_termination(active_procs(), 4):
|
||||
if not wait_for_termination(active_procs(), term_wait_s):
|
||||
# Send SIGKILL if still running
|
||||
remaining = active_procs()
|
||||
logger.warning(
|
||||
@@ -518,7 +830,13 @@ class MultiprocExecutor(Executor):
|
||||
if w.death_writer is not None:
|
||||
w.death_writer.close()
|
||||
w.death_writer = None
|
||||
self._ensure_worker_termination([w.proc for w in workers])
|
||||
self._ensure_worker_termination(
|
||||
[w.proc for w in workers],
|
||||
after_failure=getattr(self, "is_failed", False),
|
||||
shutdown_states=[
|
||||
getattr(w, "shutdown_state", None) for w in workers
|
||||
],
|
||||
)
|
||||
|
||||
for w in workers:
|
||||
# Shutdown response queues
|
||||
@@ -569,6 +887,8 @@ class UnreadyWorkerProcHandle:
|
||||
rank: int
|
||||
ready_pipe: Connection
|
||||
death_writer: Connection | None = None
|
||||
shutdown_state: Any = None
|
||||
fault_reader: Connection | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -582,6 +902,10 @@ class WorkerProcHandle:
|
||||
# `peer_worker_response_mqs[i]`
|
||||
peer_worker_response_mqs: list[MessageQueue | None]
|
||||
death_writer: Connection | None = None
|
||||
# Shared byte: _SHUTDOWN_NOT_STARTED / _STARTED / _DONE (worker writes).
|
||||
shutdown_state: Any = None
|
||||
# Read end of the worker's fault pipe (see start_worker_monitor).
|
||||
fault_reader: Connection | None = None
|
||||
|
||||
@classmethod
|
||||
def from_unready_handle(
|
||||
@@ -596,6 +920,8 @@ class WorkerProcHandle:
|
||||
worker_response_mq=worker_response_mq,
|
||||
peer_worker_response_mqs=peer_worker_response_mqs,
|
||||
death_writer=unready_handle.death_writer,
|
||||
shutdown_state=unready_handle.shutdown_state,
|
||||
fault_reader=unready_handle.fault_reader,
|
||||
)
|
||||
|
||||
|
||||
@@ -672,6 +998,10 @@ class WorkerProc:
|
||||
|
||||
# Load model
|
||||
self.worker.init_device()
|
||||
# PLE pull transport: DP0/TP0 starts the CPU-only PLE process before
|
||||
# load_model, so its weight load overlaps ours; READY is awaited after.
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
self.worker.spawn_ple_offload()
|
||||
# Update process title now that parallel groups are initialized
|
||||
self.setup_proc_title_and_log_prefix(
|
||||
enable_ep=vllm_config.parallel_config.enable_expert_parallel
|
||||
@@ -680,6 +1010,8 @@ class WorkerProc:
|
||||
self.worker.elastic_ep_execute("load_model")
|
||||
else:
|
||||
self.worker.load_model()
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
self.worker.wait_ple_offload_ready()
|
||||
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
self.use_async_scheduling = scheduler_config.async_scheduling
|
||||
@@ -715,14 +1047,31 @@ class WorkerProc:
|
||||
inherited_fds: list[int] | None = None,
|
||||
) -> UnreadyWorkerProcHandle:
|
||||
context = get_mp_context()
|
||||
# Workers arm PR_SET_PDEATHSIG against this process and verify it via
|
||||
# getppid(). Under forkserver the OS parent is the forkserver, so a
|
||||
# healthy worker would kill itself; refuse before creating anything.
|
||||
start_method = context.get_start_method()
|
||||
if start_method not in ("fork", "spawn"):
|
||||
raise RuntimeError(
|
||||
"vLLM worker processes require the 'fork' or 'spawn' "
|
||||
f"multiprocessing start method (got {start_method!r})"
|
||||
)
|
||||
# Ready pipe to communicate readiness from child to parent
|
||||
ready_reader, ready_writer = context.Pipe(duplex=False)
|
||||
# Death pipe to let child detect parent process exit
|
||||
death_reader, death_writer = context.Pipe(duplex=False)
|
||||
# Fault pipe: the worker's fail-closed path writes one byte before
|
||||
# os._exit; the executor's monitor waits on it next to the sentinel.
|
||||
fault_reader, fault_writer = context.Pipe(duplex=False)
|
||||
# One shared byte: has this worker started / finished its shutdown?
|
||||
shutdown_state = context.RawValue("b", _SHUTDOWN_NOT_STARTED)
|
||||
if inherited_fds is not None:
|
||||
inherited_fds = inherited_fds.copy()
|
||||
inherited_fds.extend((ready_reader.fileno(), death_writer.fileno()))
|
||||
inherited_fds.extend(
|
||||
(ready_reader.fileno(), death_writer.fileno(), fault_reader.fileno())
|
||||
)
|
||||
process_kwargs = {
|
||||
"expected_parent_pid": os.getpid(),
|
||||
"vllm_config": vllm_config,
|
||||
"local_rank": local_rank,
|
||||
"rank": rank,
|
||||
@@ -730,6 +1079,8 @@ class WorkerProc:
|
||||
"input_shm_handle": input_shm_handle,
|
||||
"ready_pipe": ready_writer,
|
||||
"death_pipe": death_reader,
|
||||
"fault_pipe": fault_writer,
|
||||
"shutdown_state": shutdown_state,
|
||||
"shared_worker_lock": shared_worker_lock,
|
||||
"is_driver_worker": is_driver_worker,
|
||||
# Have the worker close parent end of this worker's pipes too
|
||||
@@ -752,9 +1103,12 @@ class WorkerProc:
|
||||
# Close child ends of pipes here in the parent
|
||||
ready_writer.close()
|
||||
death_reader.close()
|
||||
fault_writer.close()
|
||||
# Keep death_writer open in parent - when parent exits,
|
||||
# death_reader in child will get EOFError
|
||||
return UnreadyWorkerProcHandle(proc, rank, ready_reader, death_writer)
|
||||
return UnreadyWorkerProcHandle(
|
||||
proc, rank, ready_reader, death_writer, shutdown_state, fault_reader
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def wait_for_response_handle_ready(
|
||||
@@ -790,10 +1144,59 @@ class WorkerProc:
|
||||
ready_proc_handles: list[WorkerProcHandle | None] = [None] * len(
|
||||
unready_proc_handles
|
||||
)
|
||||
while pipes:
|
||||
ready = multiprocessing.connection.wait(pipes.keys())
|
||||
for pipe in ready:
|
||||
assert isinstance(pipe, Connection)
|
||||
# A worker that dies before READY need not close its ready pipe: a
|
||||
# child it fork+exec'd (torch_shm_manager, started by the PLE
|
||||
# registration) can hold the write end, so no EOF ever comes. Wait on
|
||||
# each pending worker's exit (pidfd, else its sentinel -- see
|
||||
# _worker_exit_waitables) and on its fault pipe (one byte from the
|
||||
# fail-closed path; EOF is not a fault), as the post-READY monitor does.
|
||||
exit_waitables = _worker_exit_waitables(
|
||||
[h.proc for h in unready_proc_handles]
|
||||
)
|
||||
exits = {
|
||||
w: handle
|
||||
for (w, _, _), handle in zip(exit_waitables, unready_proc_handles)
|
||||
}
|
||||
faults = {
|
||||
handle.fault_reader: handle
|
||||
for handle in unready_proc_handles
|
||||
if getattr(handle, "fault_reader", None) is not None
|
||||
}
|
||||
try:
|
||||
while pipes:
|
||||
waiting: list[Any] = [*pipes.keys()]
|
||||
# EVERY worker's exit and fault pipe, READY or not: the live
|
||||
# monitor starts only after all are READY, so a READY worker
|
||||
# that faults or dies while a sibling initialises must fail
|
||||
# initialization here.
|
||||
waiting += list(exits)
|
||||
waiting += list(faults)
|
||||
ready = multiprocessing.connection.wait(waiting)
|
||||
WorkerProc._receive_ready(
|
||||
ready, pipes, exits, faults, ready_proc_handles, e
|
||||
)
|
||||
finally:
|
||||
for _, _, pidfd in exit_waitables:
|
||||
if pidfd is not None:
|
||||
with suppress(OSError):
|
||||
os.close(pidfd)
|
||||
|
||||
return cast(list[WorkerProcHandle], ready_proc_handles)
|
||||
|
||||
@staticmethod
|
||||
def _receive_ready(
|
||||
ready: list[Any],
|
||||
pipes: dict[Connection, UnreadyWorkerProcHandle],
|
||||
exits: dict[Any, UnreadyWorkerProcHandle],
|
||||
faults: dict[Any, UnreadyWorkerProcHandle],
|
||||
ready_proc_handles: list[WorkerProcHandle | None],
|
||||
e: Exception,
|
||||
) -> None:
|
||||
# Ready pipes first: a READY sent before an exit or fault still counts.
|
||||
ordered = sorted(ready, key=lambda obj: 0 if obj in pipes else 1)
|
||||
for obj in ordered:
|
||||
if obj in pipes:
|
||||
pipe = obj
|
||||
try:
|
||||
# Wait until the WorkerProc is ready.
|
||||
unready_proc_handle = pipes.pop(pipe)
|
||||
@@ -802,20 +1205,59 @@ class WorkerProc:
|
||||
raise e
|
||||
|
||||
idx = unready_proc_handle.rank % len(ready_proc_handles)
|
||||
ready_proc_handles[idx] = WorkerProc.wait_for_response_handle_ready(
|
||||
response, unready_proc_handle
|
||||
ready_proc_handles[idx] = (
|
||||
WorkerProc.wait_for_response_handle_ready(
|
||||
response, unready_proc_handle
|
||||
)
|
||||
)
|
||||
except EOFError:
|
||||
e.__suppress_context__ = True
|
||||
raise e from None
|
||||
|
||||
finally:
|
||||
# Close connection.
|
||||
pipe.close()
|
||||
continue
|
||||
handle = exits.get(obj) if obj in exits else faults.get(obj)
|
||||
if handle is None:
|
||||
continue
|
||||
pending = handle.ready_pipe in pipes
|
||||
if pending and handle.ready_pipe.poll(0):
|
||||
continue # its READY is queued: read it next round
|
||||
phase = (
|
||||
"before READY"
|
||||
if pending
|
||||
else "after READY, while other workers were initializing"
|
||||
)
|
||||
if obj in exits:
|
||||
logger.error(
|
||||
"Worker proc %s exited (code %s) %s",
|
||||
handle.proc.name,
|
||||
handle.proc.exitcode,
|
||||
phase,
|
||||
)
|
||||
raise e
|
||||
try:
|
||||
data = os.read(obj.fileno(), 1)
|
||||
except OSError:
|
||||
data = b""
|
||||
if data:
|
||||
logger.error(
|
||||
"Worker proc %s reported a fatal fault (fail-closed) %s",
|
||||
handle.proc.name,
|
||||
phase,
|
||||
)
|
||||
raise e
|
||||
# EOF on the fault pipe is not a fault; its exit follows.
|
||||
del faults[obj]
|
||||
|
||||
return cast(list[WorkerProcHandle], ready_proc_handles)
|
||||
|
||||
def shutdown(self):
|
||||
def shutdown(self, *, destroy_distributed: bool = True):
|
||||
if not destroy_distributed:
|
||||
# worker_main os._exits right after this: the PLE offload handle
|
||||
# (rank 0 owns the child) must not wait for the child's reap,
|
||||
# which trails its exit by seconds for a large host table.
|
||||
ple_worker = sys.modules.get("vllm.v1.ple_offload.worker")
|
||||
if ple_worker is not None:
|
||||
ple_worker.mark_process_exit_imminent()
|
||||
if self.rpc_broadcast_mq is not None:
|
||||
self.rpc_broadcast_mq.shutdown()
|
||||
if self.worker_response_mq is not None:
|
||||
@@ -823,8 +1265,12 @@ class WorkerProc:
|
||||
self.worker.shutdown()
|
||||
self.rpc_broadcast_mq = None
|
||||
self.worker_response_mq = None
|
||||
destroy_model_parallel()
|
||||
destroy_distributed_environment()
|
||||
# worker_main passes False when the process exits right after this
|
||||
# (see _exit_worker_process): the NCCL destroy is exit-only work that
|
||||
# can block for 5 s per communicator.
|
||||
if destroy_distributed:
|
||||
destroy_model_parallel()
|
||||
destroy_distributed_environment()
|
||||
|
||||
def monitor_death_pipe(self, death_pipe, shutdown_requested: threading.Event):
|
||||
if death_pipe is None:
|
||||
@@ -858,6 +1304,9 @@ class WorkerProc:
|
||||
if logging_config := getattr(kwargs["vllm_config"], "logging_config", None):
|
||||
configure_logging(logging_config)
|
||||
|
||||
expected_parent_pid = kwargs.pop("expected_parent_pid")
|
||||
_install_worker_parent_death_signal(expected_parent_pid)
|
||||
|
||||
# Signal handler used for graceful termination.
|
||||
# SystemExit exception is only raised once to allow this and worker
|
||||
# processes to terminate without error
|
||||
@@ -890,8 +1339,30 @@ class WorkerProc:
|
||||
set_worker_net_device(kwargs.get("local_rank", 0), kwargs["vllm_config"])
|
||||
|
||||
worker = None
|
||||
# Set once a READY worker leaves its busy loop because shutdown was
|
||||
# requested (death pipe EOF or SIGTERM/SIGINT): the process then exits
|
||||
# straight after WorkerProc.shutdown (see _exit_worker_process).
|
||||
orderly_shutdown = False
|
||||
# Set when a READY worker's busy loop FAILED (anything but its own
|
||||
# shutdown-request cancellation): full teardown, then exit code 1.
|
||||
# Never keyed to shutdown_requested -- the death-pipe monitor can set
|
||||
# it just after the check, which made a failure exit 0.
|
||||
busy_loop_failed = False
|
||||
ready_writer = kwargs.pop("ready_pipe")
|
||||
death_pipe = kwargs.pop("death_pipe", None)
|
||||
# Fault pipe: registered for the PLE fail-closed path
|
||||
# (ple_offload.fatal.notify_parent_fatal); kept open for the life of
|
||||
# the process so the byte written right before os._exit reaches the
|
||||
# executor's monitor. Closed only on the orderly way out (EOF, which
|
||||
# the monitor does not count as a fault).
|
||||
fault_pipe = kwargs.pop("fault_pipe", None)
|
||||
if fault_pipe is not None:
|
||||
set_fatal_notify_fd(fault_pipe.fileno())
|
||||
shutdown_state = kwargs.pop("shutdown_state", None)
|
||||
|
||||
def mark_shutdown(value: int) -> None:
|
||||
if shutdown_state is not None:
|
||||
shutdown_state.value = value
|
||||
|
||||
# Close inherited pipes from parent (incl. other worker pipes)
|
||||
# Explicitly passing in existing pipes and closing them makes the pipe
|
||||
@@ -902,6 +1373,8 @@ class WorkerProc:
|
||||
os.close(fd)
|
||||
except Exception as e:
|
||||
logger.warning("Error closing inherited connection: %s: %s", type(e), e)
|
||||
# Before anything below can fork+exec a helper (torch_shm_manager).
|
||||
_make_inherited_pipes_close_on_exec()
|
||||
|
||||
try:
|
||||
# Initialize tracer
|
||||
@@ -938,7 +1411,7 @@ class WorkerProc:
|
||||
|
||||
worker.worker_busy_loop()
|
||||
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
# NOTE: if an Exception arises in busy_loop, we send
|
||||
# a FAILURE message over the MQ RPC to notify the Executor,
|
||||
# which triggers system shutdown.
|
||||
@@ -946,12 +1419,23 @@ class WorkerProc:
|
||||
|
||||
if ready_writer is not None:
|
||||
logger.exception("WorkerProc failed to start.")
|
||||
elif shutdown_requested.is_set():
|
||||
elif isinstance(e, WorkerShutdownRequested) and (
|
||||
shutdown_requested.is_set()
|
||||
):
|
||||
# The busy loop left BECAUSE of the shutdown request (its
|
||||
# dequeue was cancelled), not because something failed.
|
||||
logger.debug_once(
|
||||
"[shutdown] WorkerProc: exiting after shutdown request"
|
||||
)
|
||||
orderly_shutdown = True
|
||||
else:
|
||||
logger.exception("WorkerProc failed.")
|
||||
# A real failure, whether or not a shutdown request is pending
|
||||
# (yet): full teardown, then a non-zero exit.
|
||||
if shutdown_requested.is_set():
|
||||
logger.exception("WorkerProc failed during shutdown.")
|
||||
else:
|
||||
logger.exception("WorkerProc failed.")
|
||||
busy_loop_failed = True
|
||||
|
||||
# The parent sends a SIGTERM to all worker processes if
|
||||
# any worker dies. Set this value so we don't re-throw
|
||||
@@ -965,6 +1449,7 @@ class WorkerProc:
|
||||
logger.debug_once(
|
||||
"[shutdown] WorkerProc: terminated by shutdown signal"
|
||||
)
|
||||
orderly_shutdown = ready_writer is None
|
||||
else:
|
||||
logger.warning("WorkerProc was terminated")
|
||||
# SystemExit must never be ignored
|
||||
@@ -975,14 +1460,60 @@ class WorkerProc:
|
||||
ready_writer.close()
|
||||
if death_pipe is not None:
|
||||
death_pipe.close()
|
||||
# Clean up once worker exits busy loop
|
||||
if worker is not None:
|
||||
worker.shutdown()
|
||||
# The fault fd stays registered through the whole teardown: a PLE
|
||||
# fail-closed inside worker.shutdown() must still reach the
|
||||
# executor. Unregistered after it; on the orderly
|
||||
# os._exit path the process exit closes it.
|
||||
try:
|
||||
# Clean up once worker exits busy loop
|
||||
if worker is not None:
|
||||
# Tells the executor's ladder this rank is draining, not parked
|
||||
# in a collective (see _ensure_worker_termination).
|
||||
mark_shutdown(_SHUTDOWN_STARTED)
|
||||
if orderly_shutdown:
|
||||
# Everything that must happen before exit (FINAL DRAIN,
|
||||
# instrumentation flushes, PLE cleanup) is in
|
||||
# worker.shutdown(); then exit instead of paying the NCCL
|
||||
# destroy + finalization past the executor's grace.
|
||||
exit_code = 0
|
||||
try:
|
||||
worker.shutdown(destroy_distributed=False)
|
||||
except BaseException:
|
||||
logger.exception("WorkerProc shutdown failed")
|
||||
exit_code = 1
|
||||
mark_shutdown(_SHUTDOWN_DONE)
|
||||
_exit_worker_process(exit_code)
|
||||
worker.shutdown()
|
||||
mark_shutdown(_SHUTDOWN_DONE)
|
||||
if busy_loop_failed:
|
||||
sys.exit(1)
|
||||
finally:
|
||||
if fault_pipe is not None:
|
||||
set_fatal_notify_fd(None)
|
||||
fault_pipe.close()
|
||||
|
||||
class ResponseStatus(Enum):
|
||||
SUCCESS = auto()
|
||||
FAILURE = auto()
|
||||
|
||||
def _fail_ple_rpc_if_pending(self, where: str, error: BaseException) -> None:
|
||||
"""Fail closed if RPC recovery would abandon queued PLE GPU work."""
|
||||
owners = (self.worker, getattr(self.worker, "worker", None))
|
||||
for owner in owners:
|
||||
if owner is None:
|
||||
continue
|
||||
runner = getattr(owner, "model_runner", None)
|
||||
connector = getattr(runner, "_ple_offload_connector", None)
|
||||
if connector is None:
|
||||
continue
|
||||
try:
|
||||
pending = connector.has_pending_completions
|
||||
except BaseException as state_error:
|
||||
connector._fail_closed(f"{where}:pending-state", state_error)
|
||||
if pending:
|
||||
connector._fail_closed(where, error)
|
||||
return
|
||||
|
||||
def enqueue_output(self, output: Any):
|
||||
"""Prepares output from the worker and enqueues it to the
|
||||
worker_response_mq. If the output is an Exception, it is
|
||||
@@ -993,6 +1524,7 @@ class WorkerProc:
|
||||
output = output.get_output()
|
||||
except Exception as e:
|
||||
logger.exception("Error getting async model runner output")
|
||||
self._fail_ple_rpc_if_pending("async-output-copy", e)
|
||||
output = e
|
||||
|
||||
if isinstance(output, Exception):
|
||||
@@ -1032,8 +1564,20 @@ class WorkerProc:
|
||||
def worker_busy_loop(self):
|
||||
"""Main busy loop for Multiprocessing Workers."""
|
||||
assert self.rpc_broadcast_mq is not None
|
||||
mq = self.rpc_broadcast_mq
|
||||
while True:
|
||||
self._execute_worker_rpc(self.rpc_broadcast_mq.dequeue(indefinite=True))
|
||||
# No local holds the request across the next dequeue: its payload
|
||||
# must be released first (test_multiproc_executor).
|
||||
self._execute_worker_rpc(self._dequeue_worker_rpc(mq))
|
||||
|
||||
@staticmethod
|
||||
def _dequeue_worker_rpc(mq: MessageQueue):
|
||||
try:
|
||||
return mq.dequeue(indefinite=True)
|
||||
except RuntimeError as e:
|
||||
if getattr(mq, "shutting_down", False) and str(e) == "cancelled":
|
||||
raise WorkerShutdownRequested("RPC broadcast queue shut down") from e
|
||||
raise
|
||||
|
||||
def _execute_worker_rpc(
|
||||
self,
|
||||
@@ -1052,6 +1596,7 @@ class WorkerProc:
|
||||
if output_rank is None or self.rank == output_rank:
|
||||
self.handle_output(output)
|
||||
except Exception as e:
|
||||
self._fail_ple_rpc_if_pending(f"worker-rpc:{method}", e)
|
||||
# Notes have been introduced in python 3.11
|
||||
if hasattr(e, "add_note"):
|
||||
e.add_note(traceback.format_exc())
|
||||
|
||||
@@ -67,11 +67,15 @@ class UniProcExecutor(Executor):
|
||||
|
||||
self.driver_worker.init_worker(all_kwargs=[kwargs])
|
||||
self.driver_worker.init_device()
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
self.driver_worker.spawn_ple_offload()
|
||||
|
||||
if envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH:
|
||||
self.driver_worker.elastic_ep_execute("load_model")
|
||||
else:
|
||||
self.driver_worker.load_model()
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
self.driver_worker.wait_ple_offload_ready()
|
||||
current_platform.update_block_size_for_backend(self.vllm_config)
|
||||
|
||||
def _distributed_args(self) -> tuple[str, int, int]:
|
||||
@@ -141,8 +145,13 @@ class UniProcExecutor(Executor):
|
||||
single_value=True,
|
||||
)
|
||||
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
return self.collective_rpc("take_draft_token_ids", single_value=True)
|
||||
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||
kwargs = {} if step_id is None else {"step_id": step_id}
|
||||
return self.collective_rpc(
|
||||
"take_draft_token_ids",
|
||||
kwargs=kwargs,
|
||||
single_value=True,
|
||||
)
|
||||
|
||||
def check_health(self) -> None:
|
||||
# UniProcExecutor will always be healthy as long as
|
||||
|
||||
@@ -1431,6 +1431,11 @@ class KVCacheTensor:
|
||||
block_stride: int
|
||||
offset: int = 0 # byte offset of layers[0]'s block 0
|
||||
host_resident: bool = False
|
||||
# v2.5 side-cache layout: a slot-packed ring tensor views every pool block
|
||||
# as ``slots_per_block`` equal slots (``block_stride`` is then one slot), so
|
||||
# its block ids are ``pool_block * slots_per_block + slot``. 1 = one view
|
||||
# block per pool block, the ordinary case.
|
||||
slots_per_block: int = 1
|
||||
|
||||
|
||||
class KVCacheGroupRole(str, Enum):
|
||||
@@ -1458,6 +1463,10 @@ class KVCacheGroupSpec:
|
||||
# source group after a connector load.
|
||||
enable_kv_transfer: bool = True
|
||||
role: KVCacheGroupRole = KVCacheGroupRole.DEFAULT
|
||||
# v2.5 side-cache layout: > 1 when this one-block-per-request ring group
|
||||
# shares each pool block between that many requests (see
|
||||
# ``CircularBufferManager.enable_slot_packing``); 1 = one pool block each.
|
||||
slots_per_block: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -1545,6 +1554,18 @@ class KVCacheConfig:
|
||||
)
|
||||
return tuple(block_ids[group_id] for group_id in self.transfer_group_ids)
|
||||
|
||||
@property
|
||||
def slot_packed_group_ids(self) -> tuple[int, ...]:
|
||||
"""Groups whose blocks are slots of shared pool blocks (v2.5 side-cache
|
||||
layout). Their views are not one block per pool block, so whole-block
|
||||
zeroing must skip them; their bytes lie inside pool blocks the ring
|
||||
manager owns and that no other group writes."""
|
||||
return tuple(
|
||||
group_id
|
||||
for group_id, group in enumerate(self.kv_cache_groups)
|
||||
if group.slots_per_block > 1
|
||||
)
|
||||
|
||||
@property
|
||||
def host_group_ids(self) -> tuple[int, ...]:
|
||||
return tuple(
|
||||
@@ -1556,7 +1577,7 @@ class KVCacheConfig:
|
||||
def num_blocks_of(self, tensor: KVCacheTensor) -> int:
|
||||
"""Number of blocks addressable by the pool backing ``tensor``."""
|
||||
if not tensor.host_resident:
|
||||
return self.num_blocks
|
||||
return self.num_blocks * tensor.slots_per_block
|
||||
assert self.hisparse_host_num_blocks is not None
|
||||
return self.hisparse_host_num_blocks
|
||||
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,997 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Exchange PLE data between a GPU worker and the CPU-offload process."""
|
||||
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from multiprocessing.reduction import ForkingPickler
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import zmq
|
||||
from cuda.bindings import driver as cuda_driver
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.parallel_state import get_dp_group, get_pp_group, get_tp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.ple_offload_layer import (
|
||||
CpuGpuSemaphore,
|
||||
PleOffloadLayer,
|
||||
flag_check_is_authoritative,
|
||||
stale_offload_flags,
|
||||
)
|
||||
from vllm.v1.ple_offload.fatal import notify_parent_fatal, ple_fail_closed
|
||||
from vllm.v1.ple_offload.progress import PleProgressMonitor
|
||||
from vllm.v1.ple_offload.protocol import (
|
||||
PleOffloadRegistration,
|
||||
PleOffloadRequest,
|
||||
)
|
||||
from vllm.v1.utils import record_function_or_nullcontext
|
||||
from vllm_ple_test_fault import _FAULT
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# PLE diagnostics / vllm#53899 class: read the pull-transport semaphore back before a real
|
||||
# launch and refuse to publish into a flag that is already set. Costs a
|
||||
# model-stream synchronize, so it is a diagnostic, not a hot-path guard, and is
|
||||
# OFF by default -- the same split VLLM_E2_ZERO_OWNERSHIP_CHECK established in
|
||||
# the scheduler (VLLM_E2_ZERO_OWNERSHIP_CHECK).
|
||||
#
|
||||
# off | warn | raise (1 is accepted as warn). Default off. The default action on
|
||||
# a violation is WARN, not raise: this diagnostic could
|
||||
# terminate a healthy run, and a diagnostic that kills the qualification boot
|
||||
# intended to establish safety is worse than no diagnostic. raise is for a
|
||||
# targeted repro.
|
||||
#
|
||||
# Read through the envs registry, not os.getenv: the registry owns the
|
||||
# normalization, so the value reported by
|
||||
# `vllm.envs` and the value this check applies cannot drift apart. It is
|
||||
# registered as a non-compile factor there. A set-but-unrecognised value
|
||||
# raises on first read -- fail closed, because a broken gate that silently
|
||||
# disarms its own instrument is the failure mode these checks exist to avoid.
|
||||
_PLE_FLAG_CHECK_MODE: str = envs.VLLM_E2_PLE_FLAG_CHECK
|
||||
# Liveness cadence for the diagnostic checks: the first check and every Nth
|
||||
# are logged with a running count, so "zero findings" can be told apart from
|
||||
# "never ran" in a reference-host log.
|
||||
_E2_CHECK_HEARTBEAT = 4096
|
||||
|
||||
# CUDA registration alone does not own Python storage. Captured graphs retain
|
||||
# raw addresses, so every successfully registered host allocation remains
|
||||
# strongly referenced until process exit.
|
||||
_PROCESS_LIFETIME_REGISTERED_STORAGE: list[torch.Tensor] = []
|
||||
|
||||
|
||||
def _unlink_filename_shared_storage(
|
||||
tensor: torch.Tensor,
|
||||
reserved_filenames: set[bytes],
|
||||
) -> None:
|
||||
"""Reserve and unlink one already filename-shared CPU storage."""
|
||||
try:
|
||||
storage = tensor.untyped_storage()
|
||||
if storage.device.type != "cpu" or not storage.is_shared():
|
||||
return
|
||||
try:
|
||||
storage._get_shared_fd()
|
||||
except RuntimeError:
|
||||
# Filename-shared storages do not expose a shared fd. Only now is
|
||||
# it safe to query the filename: doing so on ordinary or fd-shared
|
||||
# storage would convert it to a new filename-backed allocation.
|
||||
pass
|
||||
else:
|
||||
return
|
||||
_, filename, _ = storage._share_filename_cpu_()
|
||||
if not isinstance(filename, bytes):
|
||||
filename = os.fsencode(filename)
|
||||
path = os.path.join("/dev/shm", os.fsdecode(filename).lstrip("/"))
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to inspect a PLE shared-memory storage during teardown",
|
||||
exc_info=True,
|
||||
)
|
||||
return
|
||||
|
||||
# RefcountedMapAllocator unlinks the name when its shared counter reaches
|
||||
# zero. Reserve one count before manual unlink so later storage destruction
|
||||
# cannot try to unlink the already-removed name. The filename identifies
|
||||
# the shared counter and also deduplicates tensor aliases and repeated close.
|
||||
if filename in reserved_filenames:
|
||||
return
|
||||
try:
|
||||
storage._shared_incref()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to reserve the PLE shared-memory counter for %s",
|
||||
path,
|
||||
exc_info=True,
|
||||
)
|
||||
return
|
||||
reserved_filenames.add(filename)
|
||||
|
||||
try:
|
||||
os.unlink(path)
|
||||
except FileNotFoundError:
|
||||
# Another teardown participant already removed the name. Retain the
|
||||
# reservation because this allocator must still avoid a later unlink.
|
||||
pass
|
||||
except OSError as error:
|
||||
# The name remains available, so restore the allocator's original
|
||||
# counter and permit a later release attempt to retry.
|
||||
try:
|
||||
storage._shared_decref()
|
||||
except Exception:
|
||||
# The reservation state is uncertain after a failed private call.
|
||||
# Keep it marked to avoid adding another reservation on retry.
|
||||
logger.warning(
|
||||
"Failed to roll back the PLE shared-memory counter for %s",
|
||||
path,
|
||||
exc_info=True,
|
||||
)
|
||||
else:
|
||||
reserved_filenames.discard(filename)
|
||||
logger.warning("Failed to unlink PLE shared memory %s: %s", path, error)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _PendingPleOffloadRequest:
|
||||
"""Bind request metadata to its MRV2 D2H completion event."""
|
||||
|
||||
request: PleOffloadRequest
|
||||
d2h_done_event: torch.cuda.Event
|
||||
eligible: bool = False
|
||||
transaction_id: str | None = None
|
||||
correlation_id: str | None = None
|
||||
|
||||
|
||||
def _cuda_check(result: Any, operation: str) -> Any:
|
||||
"""Check the ``(CUresult, ...)`` tuple returned by cuda-python calls."""
|
||||
error = result[0] if isinstance(result, tuple) else result
|
||||
if error.value != 0:
|
||||
raise RuntimeError(f"{operation} failed: {error}")
|
||||
return result
|
||||
|
||||
|
||||
class PleOffloadConnector:
|
||||
"""Connect a model runner V2 GPU worker to the PLE CPU worker."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
model: nn.Module,
|
||||
device: torch.device,
|
||||
ipc_addr: str,
|
||||
*,
|
||||
input_ids_source: torch.Tensor,
|
||||
query_start_loc_source: torch.Tensor,
|
||||
ngram_context_source: torch.Tensor | None,
|
||||
) -> None:
|
||||
self.device = device
|
||||
self.rank = int(vllm_config.parallel_config.rank)
|
||||
self.dp_rank = get_dp_group().rank_in_group
|
||||
self.tp_rank = get_tp_group().rank_in_group
|
||||
self.pp_rank = get_pp_group().rank_in_group
|
||||
# Host-mapped pull design: per layer a shared host output buffer and a
|
||||
# shared host int32 flag (element 0), filled/stored by the offload
|
||||
# process with plain CPU writes. Page-locked + device-mapped here only
|
||||
# after registration, because serialising for registration may move
|
||||
# the shared storage to a new address.
|
||||
self._host_output_bufs: dict[str, torch.Tensor] = {}
|
||||
self._host_flags: dict[str, torch.Tensor] = {}
|
||||
self._reserved_host_storage_filenames: set[bytes] = set()
|
||||
self._gpu_output_bufs: dict[str, torch.Tensor] = {}
|
||||
self._layers = self._setup_layers(vllm_config, model)
|
||||
self._progress = PleProgressMonitor.from_environment(
|
||||
rank=self.rank,
|
||||
layer_names=tuple(self._layers),
|
||||
)
|
||||
|
||||
# TP0 registers shared buffers with CUDA for asynchronous D2H copies.
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
self._input_ids_buf = torch.empty(
|
||||
scheduler_config.max_num_batched_tokens,
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
).share_memory_()
|
||||
self._query_start_loc_buf = torch.empty(
|
||||
scheduler_config.max_num_seqs + 1,
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
).share_memory_()
|
||||
self._ngram_context_buf = None
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
ngram_context_len = int(config.ngram_size) - 1
|
||||
if ngram_context_len > 0:
|
||||
self._ngram_context_buf = torch.empty(
|
||||
scheduler_config.max_num_seqs,
|
||||
ngram_context_len,
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
).share_memory_()
|
||||
|
||||
# Runner input allocations are address-stable, so bind them once and
|
||||
# pass only batch sizes through the per-forward request queue.
|
||||
self._input_ids_source = input_ids_source
|
||||
self._query_start_loc_source = query_start_loc_source
|
||||
self._ngram_context_source = ngram_context_source
|
||||
self._validate_input_sources()
|
||||
|
||||
self._pinned_input_buffers: list[torch.Tensor] = []
|
||||
# Device-mapped host outputs/flags referenced by raw address from
|
||||
# captured graphs: never unregistered while the process lives
|
||||
# (unregistering under a pending graph wait unmaps it).
|
||||
self._mapped_host_buffers: list[torch.Tensor] = []
|
||||
# True while dummy/capture work (seeds, dummy forwards, capture
|
||||
# cleanup resets) has run since the last real forward. Releases in
|
||||
# that state complete synchronously, and the next real forward first
|
||||
# passes a TP barrier, so no rank can publish a real request while
|
||||
# another rank still has a dummy/cleanup reset pending.
|
||||
# In this tree real-path kernel warm-ups run BEFORE
|
||||
# capture and capture adds seeds/resets after them, so this must hold
|
||||
# at every dummy->real transition, not only at the first real one.
|
||||
self._after_dummy = True
|
||||
self._request_queue: queue.Queue[_PendingPleOffloadRequest | None] = (
|
||||
queue.Queue(maxsize=vllm_config.max_concurrent_batches)
|
||||
)
|
||||
self._dropped_events: list[torch.cuda.Event] = []
|
||||
self._request_thread: threading.Thread | None = None
|
||||
self._request_thread_ready = threading.Event()
|
||||
self._zmq_ctx: zmq.Context | None = None
|
||||
self._registration_socket: zmq.Socket | None = None
|
||||
self._d2h_event_pool: queue.Queue[torch.cuda.Event] | None = None
|
||||
|
||||
try:
|
||||
self._zmq_ctx = zmq.Context()
|
||||
self._registration_socket = self._zmq_ctx.socket(zmq.PUSH)
|
||||
self._registration_socket.connect(ipc_addr)
|
||||
payload = self._serialize_registration(vllm_config)
|
||||
|
||||
# Serialising moved the shared storages to their final addresses.
|
||||
# Page-lock/map exactly those, THEN publish, so a received
|
||||
# registration always means "storage usable".
|
||||
with torch.accelerator.device_index(self.device.index):
|
||||
self._map_host_outputs(vllm_config)
|
||||
if self.tp_rank == 0:
|
||||
self._pin_input_buffers()
|
||||
self._send_registration(payload, ipc_addr)
|
||||
|
||||
if self.tp_rank == 0:
|
||||
with torch.accelerator.device_index(self.device.index):
|
||||
self._d2h_event_pool = queue.Queue(
|
||||
maxsize=vllm_config.max_concurrent_batches
|
||||
)
|
||||
for _ in range(vllm_config.max_concurrent_batches):
|
||||
self._d2h_event_pool.put_nowait(torch.cuda.Event())
|
||||
self._start_request_thread(ipc_addr)
|
||||
except Exception:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def _setup_layers(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
model: nn.Module,
|
||||
) -> dict[str, PleOffloadLayer]:
|
||||
"""Attach output buffers and semaphores to GPU PLE placeholders."""
|
||||
layers = {
|
||||
name: module
|
||||
for name, module in model.named_modules()
|
||||
if isinstance(module, PleOffloadLayer)
|
||||
}
|
||||
if not layers:
|
||||
raise RuntimeError(
|
||||
"VLLM_E2_PLE_PULL_TRANSPORT is enabled, but the model has no "
|
||||
"PleOffloadLayer"
|
||||
)
|
||||
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
for name, layer in layers.items():
|
||||
if vllm_config.load_config.load_format == "dummy":
|
||||
layer.initialize_dummy_offload_metadata(self.device)
|
||||
out_dim = layer.get_offload_output_dim(int(config.ple_embed_dim))
|
||||
out_dtype = layer.get_offload_output_dtype(vllm_config.model_config.dtype)
|
||||
# The captured forward DMAs the host result into this device
|
||||
# buffer after the host flag is set; consumers read it as before.
|
||||
self._gpu_output_bufs[name] = torch.empty(
|
||||
max_num_tokens, out_dim, dtype=out_dtype, device=self.device
|
||||
)
|
||||
# Zero-initialised so dummy/profile forwards copy finite values.
|
||||
self._host_output_bufs[name] = torch.zeros(
|
||||
max_num_tokens, out_dim, dtype=out_dtype, device="cpu"
|
||||
).share_memory_()
|
||||
# 16 x int32 = one 64-byte line; element 0 is the flag.
|
||||
self._host_flags[name] = torch.zeros(
|
||||
16, dtype=torch.int32, device="cpu"
|
||||
).share_memory_()
|
||||
return layers
|
||||
|
||||
def _map_host_outputs(self, vllm_config: VllmConfig) -> None:
|
||||
"""Page-lock + device-map host outputs/flags and arm the placeholders."""
|
||||
for name, layer in self._layers.items():
|
||||
layer._ple_offload_name = name
|
||||
host_out = self._host_output_bufs[name]
|
||||
host_flag = self._host_flags[name]
|
||||
for buffer in (host_out, host_flag):
|
||||
self._register_host_buffer(buffer, device_map=True)
|
||||
flag_dev_ptr = _cuda_check(
|
||||
cuda_driver.cuMemHostGetDevicePointer(host_flag.data_ptr(), 0),
|
||||
"cuMemHostGetDevicePointer(PLE host flag)",
|
||||
)[1]
|
||||
out_dev_ptr = _cuda_check(
|
||||
cuda_driver.cuMemHostGetDevicePointer(host_out.data_ptr(), 0),
|
||||
"cuMemHostGetDevicePointer(PLE host output)",
|
||||
)[1]
|
||||
row_bytes = host_out.shape[1] * host_out.element_size()
|
||||
gpu_buf = self._gpu_output_bufs[name]
|
||||
if (
|
||||
gpu_buf.shape[0] != host_out.shape[0]
|
||||
or gpu_buf.shape[1] * gpu_buf.element_size() != row_bytes
|
||||
):
|
||||
raise RuntimeError(f"PLE host/GPU buffer geometry mismatch for {name}")
|
||||
# Stream memops use the flag's DEVICE pointer; the H2D copy uses
|
||||
# this worker's HOST pointer (no raw addresses cross
|
||||
# processes; the offload writes through its own mapping).
|
||||
layer.setup_cross_process_offload(
|
||||
gpu_buf,
|
||||
CpuGpuSemaphore.from_host_mapped(host_flag, int(flag_dev_ptr)),
|
||||
host_output_ptr=host_out.data_ptr(),
|
||||
host_row_bytes=row_bytes,
|
||||
)
|
||||
logger.info(
|
||||
"PleOffload: host-mapped pull for %s (rows %d x %d B, flag dev "
|
||||
"0x%x, out dev 0x%x, host==dev %s)",
|
||||
name,
|
||||
host_out.shape[0],
|
||||
row_bytes,
|
||||
int(flag_dev_ptr),
|
||||
int(out_dev_ptr),
|
||||
int(out_dev_ptr) == host_out.data_ptr(),
|
||||
)
|
||||
|
||||
def _pin_input_buffers(self) -> None:
|
||||
"""Page-lock shared input allocations without replacing their storage."""
|
||||
buffers = [self._input_ids_buf, self._query_start_loc_buf]
|
||||
if self._ngram_context_buf is not None:
|
||||
buffers.append(self._ngram_context_buf)
|
||||
for buffer in buffers:
|
||||
self._register_host_buffer(buffer, device_map=False)
|
||||
|
||||
def _register_host_buffer(self, buffer: torch.Tensor, device_map: bool) -> None:
|
||||
"""Page-lock (and optionally device-map) a shared CPU allocation."""
|
||||
if buffer.device.type != "cpu" or not buffer.is_shared():
|
||||
raise RuntimeError("PLE host buffers must be shared CPU tensors")
|
||||
if not buffer.is_contiguous():
|
||||
raise RuntimeError("PLE host buffers must be contiguous")
|
||||
flags = cuda_driver.CU_MEMHOSTREGISTER_PORTABLE
|
||||
if device_map:
|
||||
flags |= cuda_driver.CU_MEMHOSTREGISTER_DEVICEMAP
|
||||
_cuda_check(
|
||||
cuda_driver.cuMemHostRegister(
|
||||
buffer.data_ptr(),
|
||||
buffer.numel() * buffer.element_size(),
|
||||
flags,
|
||||
),
|
||||
"cuMemHostRegister(PLE host buffer)",
|
||||
)
|
||||
_PROCESS_LIFETIME_REGISTERED_STORAGE.append(buffer)
|
||||
if device_map:
|
||||
self._mapped_host_buffers.append(buffer)
|
||||
else:
|
||||
self._pinned_input_buffers.append(buffer)
|
||||
if not buffer.is_pinned():
|
||||
raise RuntimeError("CUDA did not page-lock a PLE host buffer")
|
||||
|
||||
def _unpin_input_buffers(self) -> None:
|
||||
"""Release CUDA registrations after the request thread has stopped."""
|
||||
for buffer in reversed(self._pinned_input_buffers):
|
||||
try:
|
||||
_cuda_check(
|
||||
cuda_driver.cuMemHostUnregister(buffer.data_ptr()),
|
||||
"cuMemHostUnregister(PLE input buffer)",
|
||||
)
|
||||
except RuntimeError:
|
||||
logger.exception("Failed to unregister a PLE input buffer")
|
||||
self._pinned_input_buffers.clear()
|
||||
|
||||
def _serialize_registration(self, vllm_config: VllmConfig) -> Any:
|
||||
"""Build and serialise the registration (may move shared storages)."""
|
||||
# Each GPU worker owns distinct host output buffers and flags, while
|
||||
# TP0's shared inputs become the request source for its DP rank.
|
||||
registration = PleOffloadRegistration(
|
||||
worker_id=(
|
||||
self.dp_rank * vllm_config.parallel_config.world_size
|
||||
+ vllm_config.parallel_config.rank
|
||||
),
|
||||
tp_rank=self.tp_rank,
|
||||
dp_rank=self.dp_rank,
|
||||
host_output_buffers=dict(self._host_output_bufs),
|
||||
host_flag_tensors=dict(self._host_flags),
|
||||
input_ids_buf=self._input_ids_buf,
|
||||
query_start_loc_buf=self._query_start_loc_buf,
|
||||
ngram_context_buf=self._ngram_context_buf,
|
||||
)
|
||||
|
||||
# ForkingPickler transmits tensors through shared-memory and CUDA IPC.
|
||||
import torch.multiprocessing as torch_mp
|
||||
|
||||
original_strategy = torch_mp.get_sharing_strategy()
|
||||
torch_mp.set_sharing_strategy("file_system")
|
||||
try:
|
||||
payload = ForkingPickler.dumps(registration)
|
||||
finally:
|
||||
torch_mp.set_sharing_strategy(original_strategy)
|
||||
return payload
|
||||
|
||||
def _send_registration(self, payload: Any, ipc_addr: str) -> None:
|
||||
"""Publish the serialised registration (storages already mapped)."""
|
||||
assert self._registration_socket is not None
|
||||
self._registration_socket.send(payload)
|
||||
|
||||
logger.info(
|
||||
"PleOffload: registered %d PleOffloadLayer(s) "
|
||||
"(dp_rank=%d, tp_rank=%d, ipc_addr=%s): %s",
|
||||
len(self._layers),
|
||||
self.dp_rank,
|
||||
self.tp_rank,
|
||||
ipc_addr,
|
||||
sorted(self._layers),
|
||||
)
|
||||
|
||||
def _start_request_thread(self, ipc_addr: str) -> None:
|
||||
"""Start the thread that publishes batches after inputs are ready."""
|
||||
self._request_thread = threading.Thread(
|
||||
target=self._request_loop,
|
||||
args=(ipc_addr,),
|
||||
name=f"ple-offload-dp{self.dp_rank}",
|
||||
daemon=True,
|
||||
)
|
||||
self._request_thread.start()
|
||||
if not self._request_thread_ready.wait(timeout=10):
|
||||
raise RuntimeError("Timed out starting the PLE request thread")
|
||||
|
||||
def _request_loop(self, ipc_addr: str) -> None:
|
||||
"""Wait for staged inputs, then notify the CPU worker."""
|
||||
socket: zmq.Socket | None = None
|
||||
try:
|
||||
if self._zmq_ctx is None:
|
||||
raise RuntimeError("PLE ZMQ context closed before thread startup")
|
||||
socket = self._zmq_ctx.socket(zmq.PUSH)
|
||||
socket.connect(ipc_addr)
|
||||
self._request_thread_ready.set()
|
||||
while True:
|
||||
request = self._request_queue.get()
|
||||
if request is None:
|
||||
return
|
||||
self._process_request(request, socket)
|
||||
except Exception:
|
||||
logger.exception("PLE request thread failed")
|
||||
notify_parent_fatal()
|
||||
os._exit(1)
|
||||
finally:
|
||||
self._request_thread_ready.set()
|
||||
if socket is not None:
|
||||
socket.close(linger=0)
|
||||
|
||||
def _process_request(
|
||||
self, pending: _PendingPleOffloadRequest, socket: zmq.Socket
|
||||
) -> None:
|
||||
"""Wait for one staged batch and publish its request."""
|
||||
request = pending.request
|
||||
event_pool = self._d2h_event_pool
|
||||
event = pending.d2h_done_event
|
||||
assert event_pool is not None, "PLE D2H event pool is not initialized"
|
||||
if _FAULT is not None and _FAULT.fires(
|
||||
"block_input_completion",
|
||||
role="gpu",
|
||||
unit="connector-request",
|
||||
eligible=pending.eligible,
|
||||
tp_rank=self.tp_rank,
|
||||
pp_rank=self.pp_rank,
|
||||
dp_rank=self.dp_rank,
|
||||
transaction_id=pending.transaction_id,
|
||||
correlation_id=pending.correlation_id,
|
||||
):
|
||||
threading.Event().wait()
|
||||
|
||||
with torch.accelerator.device_index(self.device.index):
|
||||
event.synchronize()
|
||||
|
||||
payload = msgspec.msgpack.encode(request)
|
||||
if _FAULT is None:
|
||||
socket.send(payload)
|
||||
else:
|
||||
# Armed runs only: correlation metadata rides as a second frame.
|
||||
metadata = msgspec.msgpack.encode(
|
||||
{
|
||||
"serving": pending.eligible,
|
||||
"transaction": pending.transaction_id,
|
||||
"correlation": pending.correlation_id,
|
||||
}
|
||||
)
|
||||
socket.send_multipart([payload, metadata])
|
||||
event_pool.put_nowait(event)
|
||||
|
||||
def _validate_input_sources(self) -> None:
|
||||
"""Validate fixed runner sources against shared input buffers."""
|
||||
sources = [
|
||||
("input_ids", self._input_ids_source, self._input_ids_buf),
|
||||
(
|
||||
"query_start_loc",
|
||||
self._query_start_loc_source,
|
||||
self._query_start_loc_buf,
|
||||
),
|
||||
]
|
||||
if (self._ngram_context_source is None) != (self._ngram_context_buf is None):
|
||||
raise ValueError("PLE ngram_context source and buffer must match")
|
||||
if self._ngram_context_source is not None:
|
||||
assert self._ngram_context_buf is not None
|
||||
sources.append(
|
||||
(
|
||||
"ngram_context",
|
||||
self._ngram_context_source,
|
||||
self._ngram_context_buf,
|
||||
)
|
||||
)
|
||||
|
||||
for name, source, buffer in sources:
|
||||
if (
|
||||
source.device != self.device
|
||||
or source.dtype != buffer.dtype
|
||||
or source.ndim != buffer.ndim
|
||||
or source.shape[0] < buffer.shape[0]
|
||||
or source.shape[1:] != buffer.shape[1:]
|
||||
):
|
||||
raise ValueError(f"PLE {name} source is incompatible")
|
||||
|
||||
def _enqueue_cuda_inputs(
|
||||
self,
|
||||
request: PleOffloadRequest,
|
||||
d2h_done_event: torch.cuda.Event,
|
||||
) -> None:
|
||||
"""Stage inputs on the model stream and record D2H completion."""
|
||||
with (
|
||||
record_function_or_nullcontext("ple_offload: enqueue_cuda_inputs"),
|
||||
torch.accelerator.device_index(self.device.index),
|
||||
):
|
||||
stream = torch.cuda.current_stream(self.device)
|
||||
self._input_ids_buf[: request.num_tokens].copy_(
|
||||
self._input_ids_source[: request.num_tokens],
|
||||
non_blocking=True,
|
||||
)
|
||||
self._query_start_loc_buf[: request.num_reqs + 1].copy_(
|
||||
self._query_start_loc_source[: request.num_reqs + 1],
|
||||
non_blocking=True,
|
||||
)
|
||||
if self._ngram_context_buf is not None:
|
||||
assert self._ngram_context_source is not None
|
||||
self._ngram_context_buf[: request.num_reqs].copy_(
|
||||
self._ngram_context_source[: request.num_reqs],
|
||||
non_blocking=True,
|
||||
)
|
||||
d2h_done_event.record(stream)
|
||||
|
||||
def _launch(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
*,
|
||||
serving: bool = False,
|
||||
transaction_id: str | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Stage or queue one batch for request publication."""
|
||||
# Inputs are replicated across TP ranks. One request per DP rank drives
|
||||
# the CPU result fan-out to every registered TP output buffer.
|
||||
if self.tp_rank != 0:
|
||||
return
|
||||
|
||||
request = PleOffloadRequest(
|
||||
dp_rank=self.dp_rank,
|
||||
num_tokens=num_tokens,
|
||||
num_reqs=num_reqs,
|
||||
)
|
||||
assert self._d2h_event_pool is not None, "PLE D2H event pool is not initialized"
|
||||
try:
|
||||
d2h_done_event = self._d2h_event_pool.get_nowait()
|
||||
except queue.Empty as exc:
|
||||
raise RuntimeError(
|
||||
"PLE has more requests than configured concurrent batches"
|
||||
) from exc
|
||||
self._enqueue_cuda_inputs(request, d2h_done_event)
|
||||
if _FAULT is not None and _FAULT.fires(
|
||||
"drop_request",
|
||||
role="gpu",
|
||||
unit="connector-request",
|
||||
eligible=serving,
|
||||
tp_rank=self.tp_rank,
|
||||
pp_rank=self.pp_rank,
|
||||
dp_rank=self.dp_rank,
|
||||
transaction_id=transaction_id,
|
||||
correlation_id=correlation_id,
|
||||
):
|
||||
# Deliberately retain the event: the fault is loss after D2H, not
|
||||
# premature event-pool reuse while that D2H may still be in flight.
|
||||
self._dropped_events.append(d2h_done_event)
|
||||
return
|
||||
self._request_queue.put_nowait(
|
||||
_PendingPleOffloadRequest(
|
||||
request,
|
||||
d2h_done_event,
|
||||
eligible=serving,
|
||||
transaction_id=transaction_id,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
)
|
||||
|
||||
def _fail_closed(self, where: str, error: BaseException) -> None:
|
||||
ple_fail_closed(
|
||||
where=where,
|
||||
rank=self.rank,
|
||||
layer=",".join(self._layers),
|
||||
sequence=self._progress.sequence,
|
||||
error=error,
|
||||
)
|
||||
|
||||
@property
|
||||
def has_pending_completions(self) -> bool:
|
||||
return self._progress.has_pending_completions
|
||||
|
||||
def begin_forward(self, *, dummy_run: bool, num_tokens: int) -> int:
|
||||
"""Begin the bounded transaction before any runner fault boundary."""
|
||||
try:
|
||||
return self._progress.begin(
|
||||
startup=dummy_run,
|
||||
num_tokens=num_tokens,
|
||||
)
|
||||
except Exception as error:
|
||||
self._fail_closed("prepare_forward", error)
|
||||
|
||||
def fault_transaction(self, sequence: int) -> str:
|
||||
"""Read diagnostic state only when the caller has established arming."""
|
||||
return self._progress.describe_submission(sequence)
|
||||
|
||||
def completion_is_pending(self, sequence: int) -> bool:
|
||||
"""Return exact-sequence incomplete completion state for cut 6."""
|
||||
return self._progress.completion_is_pending(sequence)
|
||||
|
||||
def prepare_forward(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
dummy_run: bool,
|
||||
*,
|
||||
consumer_num_tokens: int | None = None,
|
||||
serving: bool = False,
|
||||
transaction_id: str | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Prepare one PLE transaction; any ordinary exception is fatal."""
|
||||
try:
|
||||
self._prepare_forward_impl(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
dummy_run,
|
||||
consumer_num_tokens=consumer_num_tokens,
|
||||
serving=serving,
|
||||
transaction_id=transaction_id,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
except Exception as error:
|
||||
self._fail_closed("prepare_forward", error)
|
||||
|
||||
def _prepare_forward_impl(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
dummy_run: bool,
|
||||
*,
|
||||
consumer_num_tokens: int | None = None,
|
||||
serving: bool = False,
|
||||
transaction_id: str | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Submit real inputs or satisfy the PLE wait for a dummy forward."""
|
||||
if consumer_num_tokens is None:
|
||||
consumer_num_tokens = num_tokens
|
||||
if int(num_tokens) != int(consumer_num_tokens):
|
||||
raise RuntimeError(
|
||||
"PLE request/consumer token-count mismatch: "
|
||||
f"request={num_tokens}, consumer={consumer_num_tokens}"
|
||||
)
|
||||
if not 0 <= int(num_tokens) <= self._input_ids_buf.shape[0]:
|
||||
raise RuntimeError(
|
||||
f"PLE token count {num_tokens} exceeds connector capacity "
|
||||
f"{self._input_ids_buf.shape[0]}"
|
||||
)
|
||||
if not 0 <= int(num_reqs) < self._query_start_loc_buf.shape[0]:
|
||||
raise RuntimeError(
|
||||
f"PLE request count {num_reqs} exceeds connector capacity "
|
||||
f"{self._query_start_loc_buf.shape[0] - 1}"
|
||||
)
|
||||
if dummy_run:
|
||||
self._signal_dummy_outputs_impl(num_tokens)
|
||||
return
|
||||
if self._after_dummy:
|
||||
# Every dummy/cleanup reset on THIS rank already completed
|
||||
# (synchronous releases); the barrier extends that to every
|
||||
# stage-0 TP rank before TP0 can publish (CPU group, no GPU work;
|
||||
# all TP ranks run the same forwards in the same order).
|
||||
get_tp_group().barrier()
|
||||
self._after_dummy = False
|
||||
logger.info(
|
||||
"PleOffload: dummy->real transition (dp_rank=%d, tp_rank=%d): "
|
||||
"releases were synchronous, TP barrier passed",
|
||||
self.dp_rank,
|
||||
self.tp_rank,
|
||||
)
|
||||
if _PLE_FLAG_CHECK_MODE != "off":
|
||||
self._check_flags_clear_before_launch(num_tokens)
|
||||
self._launch(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
serving=serving,
|
||||
transaction_id=transaction_id,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
|
||||
def _check_flags_clear_before_launch(self, num_tokens: int) -> None:
|
||||
"""Test the transport's staleness precondition, off by default.
|
||||
|
||||
A real launch publishes rows that the forward will pick up with
|
||||
``cuStreamWaitValue32(flag == DONE_VALUE)``
|
||||
(``ple_offload_layer.py:279``). That wait is only meaningful if the
|
||||
flag is CLEAR when we publish: a leftover ``DONE_VALUE`` satisfies it
|
||||
from the PREVIOUS step's copy and the forward silently consumes the
|
||||
previous step's PLE outputs -- vllm#53899, where the semaphore stayed
|
||||
one step ahead for the life of the server and ``cudagraph_mode=NONE``
|
||||
was the only correct configuration.
|
||||
|
||||
Nothing here can be inferred from ``_after_dummy``: that is a bool
|
||||
tracking a transition, not the flag's value. The structural argument
|
||||
(dummy signal consumed by the dummy forward, ``_release_outputs_impl``
|
||||
resetting every layer and synchronizing while ``_after_dummy`` is set,
|
||||
a TP barrier at the transition) holds only while every
|
||||
``prepare_forward`` is matched by a ``release_outputs``. An exception
|
||||
path, an aborted step, or a future capture that signals without a
|
||||
consuming forward leaves the flag at 1, and the #53899 thread's own
|
||||
trap applies: identical sequential requests produce two bit-identical
|
||||
classes, so a "same prompt twice" check passes in the broken world
|
||||
(an instrument must discriminate both worlds).
|
||||
|
||||
PUBLISHING RANK ONLY. ``flag == 0`` before a launch is
|
||||
NOT an invariant on a non-publishing rank, and the first version of
|
||||
this check could terminate a HEALTHY TP2 run because of it. ``_launch``
|
||||
publishes only on TP0 (``if self.tp_rank != 0: return``), and the
|
||||
offload worker then fans the result out to EVERY registered target,
|
||||
each gated only on that target's own flag having been reset
|
||||
(``worker.py``, ``for target in targets: _wait_flag_released(target)
|
||||
... target.host_flag[0] = 1``). So this legal interleaving exists:
|
||||
|
||||
1. TP0 checks its flag (clear), publishes step N.
|
||||
2. The worker fills TP1's buffer and sets TP1's flag for step N.
|
||||
3. TP1 reaches its own check for step N and reads ``DONE``.
|
||||
4. TP1 reports a stale signal -- for a signal that belongs to the
|
||||
CURRENT step.
|
||||
|
||||
The dummy->real TP barrier is before the checks and does not order this.
|
||||
On TP0 the property does hold, and by construction rather than by
|
||||
timing: the worker learns of a request only from ``_launch``, which is
|
||||
after this check, and ``PleProgressMonitor.begin`` refuses overlapping
|
||||
serving submissions ("Overlapping PLE host submissions violate the
|
||||
serialized single-slot protocol"), so at most one transaction is
|
||||
outstanding and the stream synchronize below has already retired its
|
||||
release. A non-publishing rank cannot distinguish "this step's fan-out
|
||||
arrived early" from "last step's signal was never consumed" with a
|
||||
BINARY flag -- that is the same representational limit that rules out
|
||||
an epoch word (see ``ple_offload_layer.py``), so those ranks are
|
||||
skipped rather than checked weakly.
|
||||
|
||||
VLLM_E2_PLE_FLAG_CHECK=off|warn|raise, default off, default ACTION warn.
|
||||
|
||||
SCOPE. The skip below tests whether a capture
|
||||
is ACTIVE, which is not the same as "this step's forward is eager". A
|
||||
captured replay is not an active capture, and ``prepare_forward`` is
|
||||
host Python that runs ahead of the forward either way, so this check
|
||||
DOES run for steps whose forward is a graph replay. What it does not do
|
||||
is run *inside* a capture, where a host synchronize is illegal. The
|
||||
earlier "eager only" wording confused those two lifecycle states.
|
||||
|
||||
LIVENESS (PLE diagnostics on 0.30). A zero-findings result is only evidence if the
|
||||
check demonstrably RAN, so every rank says once what it does with the
|
||||
check, and the publishing rank logs a running count of checks and
|
||||
findings on its first check and every ``_E2_CHECK_HEARTBEAT`` checks
|
||||
after. A reference-host test arm greps these lines; their absence voids the arm.
|
||||
"""
|
||||
if not flag_check_is_authoritative(self.tp_rank):
|
||||
if not getattr(self, "_flag_check_skip_logged", False):
|
||||
self._flag_check_skip_logged = True
|
||||
logger.info(
|
||||
"PLE flag check (mode=%s): non-publishing rank, not judged "
|
||||
"here by design (dp_rank=%d, tp_rank=%d, pp_rank=%d)",
|
||||
_PLE_FLAG_CHECK_MODE,
|
||||
self.dp_rank,
|
||||
self.tp_rank,
|
||||
self.pp_rank,
|
||||
)
|
||||
return
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
return
|
||||
stream = torch.cuda.current_stream(self.device)
|
||||
# Order the read against any reset still queued on the model stream.
|
||||
# Without this the check reports the pre-reset value and fires on a
|
||||
# correct run -- a guard that fires on correct metadata is worse than
|
||||
# no guard.
|
||||
stream.synchronize()
|
||||
stale = stale_offload_flags(self._layers)
|
||||
runs = self._flag_checks_run = getattr(self, "_flag_checks_run", 0) + 1
|
||||
findings = getattr(self, "_flag_check_findings", 0) + (1 if stale else 0)
|
||||
self._flag_check_findings = findings
|
||||
if runs == 1 or runs % _E2_CHECK_HEARTBEAT == 0:
|
||||
logger.info(
|
||||
"PLE flag check (mode=%s): %d checks run, %d with findings "
|
||||
"(dp_rank=%d, tp_rank=%d, pp_rank=%d)",
|
||||
_PLE_FLAG_CHECK_MODE,
|
||||
runs,
|
||||
findings,
|
||||
self.dp_rank,
|
||||
self.tp_rank,
|
||||
self.pp_rank,
|
||||
)
|
||||
if not stale:
|
||||
return
|
||||
message = (
|
||||
"PLE pull transport: the output semaphore was already signalled "
|
||||
"before this publishing rank published a real request, so the "
|
||||
"forward's cuStreamWaitValue32(flag == 1) would be satisfied by "
|
||||
"the PREVIOUS step's copy and this forward would consume the "
|
||||
"previous step's PLE rows (vllm#53899 one-step-lag class). Stale "
|
||||
f"layers (name, flag): {stale}; expected "
|
||||
f"{CpuGpuSemaphore.RESET_VALUE} for all {len(self._layers)} "
|
||||
f"layer(s). num_tokens={num_tokens}, dp_rank={self.dp_rank}, "
|
||||
f"tp_rank={self.tp_rank}, pp_rank={self.pp_rank}."
|
||||
)
|
||||
if _PLE_FLAG_CHECK_MODE == "raise":
|
||||
# Reaches _fail_closed through prepare_forward, which is the
|
||||
# transport's stated contract for an uncertain correspondence.
|
||||
raise RuntimeError(message)
|
||||
logger.warning("%s", message)
|
||||
|
||||
def signal_dummy_outputs(self, num_tokens: int) -> None:
|
||||
"""Locally satisfy PLE waits for dummy and capture forwards."""
|
||||
self._progress.begin(startup=True, num_tokens=num_tokens)
|
||||
self._signal_dummy_outputs_impl(num_tokens)
|
||||
|
||||
def _signal_dummy_outputs_impl(self, num_tokens: int) -> None:
|
||||
"""Signal dummy outputs after the progress transaction is armed."""
|
||||
# Dummy and capture forwards do not send CPU requests, but every PLE
|
||||
# placeholder still waits for a completed output semaphore.
|
||||
self._after_dummy = True
|
||||
stream = torch.cuda.current_stream(self.device)
|
||||
for layer in self._layers.values():
|
||||
layer._gpu_output_buffer[:num_tokens].zero_()
|
||||
layer._sem.signal(stream)
|
||||
|
||||
def release_outputs(
|
||||
self,
|
||||
*,
|
||||
serving: bool = False,
|
||||
transaction_id: str | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Release one PLE transaction; any ordinary exception is fatal."""
|
||||
try:
|
||||
self._release_outputs_impl(
|
||||
serving=serving,
|
||||
transaction_id=transaction_id,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
self._progress.complete()
|
||||
except Exception as error:
|
||||
self._fail_closed("release_outputs", error)
|
||||
|
||||
def _release_outputs_impl(
|
||||
self,
|
||||
*,
|
||||
serving: bool = False,
|
||||
transaction_id: str | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Mark GPU output buffers reusable after the model consumes them."""
|
||||
if _FAULT is not None and _FAULT.fires(
|
||||
"exc_in_release",
|
||||
role="gpu",
|
||||
unit="connector-release",
|
||||
eligible=serving,
|
||||
tp_rank=self.tp_rank,
|
||||
pp_rank=self.pp_rank,
|
||||
dp_rank=self.dp_rank,
|
||||
transaction_id=transaction_id,
|
||||
correlation_id=correlation_id,
|
||||
):
|
||||
raise RuntimeError("PLE test fault: exc_in_release")
|
||||
# Reset only after the consumer forward so the CPU worker cannot
|
||||
# overwrite an output that a GPU PLE placeholder may still read.
|
||||
stream = torch.cuda.current_stream(self.device)
|
||||
for layer in self._layers.values():
|
||||
layer.release_offloaded_output(stream)
|
||||
# Retain the serving deadline until the model stream has actually
|
||||
# executed every reset above. The monitor thread is the sole query site.
|
||||
completion_event = torch.cuda.Event()
|
||||
completion_event.record(stream)
|
||||
self._progress.track_completion(completion_event)
|
||||
# After dummy/capture work (profile, warm-up sizes, capture seed,
|
||||
# adaptive-verification dummy replays, capture cleanup): complete the
|
||||
# reset before returning, so it cannot still be pending when a real
|
||||
# request lets the offload publish. Real-forward releases stay async.
|
||||
# Never inside a stream capture.
|
||||
if self._after_dummy and not torch.cuda.is_current_stream_capturing():
|
||||
stream.synchronize()
|
||||
|
||||
def release_host_storage(self) -> None:
|
||||
"""Unlink host-storage names while retaining mappings and tensors."""
|
||||
buffers = [
|
||||
*self._host_output_bufs.values(),
|
||||
*self._host_flags.values(),
|
||||
self._input_ids_buf,
|
||||
self._query_start_loc_buf,
|
||||
]
|
||||
if self._ngram_context_buf is not None:
|
||||
buffers.append(self._ngram_context_buf)
|
||||
for buffer in buffers:
|
||||
_unlink_filename_shared_storage(
|
||||
buffer,
|
||||
self._reserved_host_storage_filenames,
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Stop transport and unlink names, retaining host mappings."""
|
||||
request_thread = self._request_thread
|
||||
if request_thread is not None and request_thread.is_alive():
|
||||
try:
|
||||
self._request_queue.put(None, timeout=5)
|
||||
except queue.Full:
|
||||
logger.error("Timed out stopping the PLE request thread")
|
||||
request_thread.join(timeout=5)
|
||||
if request_thread is not None and request_thread.is_alive():
|
||||
# The thread may still access the registered buffers or ZMQ context.
|
||||
# It may also still race another process opening a shared name, so
|
||||
# this deferred-cleanup branch must not unlink host storage.
|
||||
logger.error("PLE request thread did not stop; deferring resource cleanup")
|
||||
self._progress.close()
|
||||
return
|
||||
self._request_thread = None
|
||||
|
||||
# Inputs, outputs, and flags remain registered and strongly referenced.
|
||||
# A captured graph or delayed request can still contain their addresses;
|
||||
# the CUDA driver releases registrations when the process exits.
|
||||
registered = len(self._pinned_input_buffers) + len(self._mapped_host_buffers)
|
||||
if registered:
|
||||
logger.debug(
|
||||
"PleOffload: %d registered host buffers retained until exit",
|
||||
registered,
|
||||
)
|
||||
self._d2h_event_pool = None
|
||||
if self._registration_socket is not None:
|
||||
self._registration_socket.close(linger=0)
|
||||
self._registration_socket = None
|
||||
if self._zmq_ctx is not None:
|
||||
self._zmq_ctx.term()
|
||||
self._zmq_ctx = None
|
||||
self._progress.close()
|
||||
|
||||
# __init__'s failure handler can reach here before the child has rebuilt
|
||||
# a registration. Startup is already failing in that case; if a payload
|
||||
# was sent, a late rebuild may also observe the removed name. Successful
|
||||
# startup accepts every registration before an external close begins.
|
||||
#
|
||||
# NOTE: Graceful shutdown unlinks the PLE shared-memory names and the
|
||||
# PLE ipc socket. The remaining 17 /tmp UUID sockets per boot are
|
||||
# upstream vLLM VLLM_RPC_BASE_PATH behaviour; a tmpfiles/launch-profile
|
||||
# sweep is recommended.
|
||||
# Keep this last: unlinking preserves every existing CPU/CUDA mapping,
|
||||
# but no transport user may still need to open a name at this point.
|
||||
self.release_host_storage()
|
||||
@@ -0,0 +1,304 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fail-closed helpers for the PLE single-slot transport."""
|
||||
|
||||
import functools
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from types import TracebackType
|
||||
from typing import Any, NoReturn, TypeVar
|
||||
|
||||
_F = TypeVar("_F", bound=Callable[..., Any])
|
||||
|
||||
_TRUSTED_ERROR_LABELS = (
|
||||
(RuntimeError, "RuntimeError"),
|
||||
(TimeoutError, "TimeoutError"),
|
||||
(ValueError, "ValueError"),
|
||||
(OSError, "OSError"),
|
||||
(EOFError, "EOFError"),
|
||||
(BrokenPipeError, "BrokenPipeError"),
|
||||
(AssertionError, "AssertionError"),
|
||||
(KeyError, "KeyError"),
|
||||
(IndexError, "IndexError"),
|
||||
(TypeError, "TypeError"),
|
||||
(MemoryError, "MemoryError"),
|
||||
(KeyboardInterrupt, "KeyboardInterrupt"),
|
||||
)
|
||||
|
||||
# Slot readers for the dying path. Each is the C getter of the base type,
|
||||
# called directly, so no __getattribute__, property, metaclass attribute or
|
||||
# descriptor defined by the exception (or its class) can run. The traceback
|
||||
# module is NOT safe here: traceback.format_exception calls str(error) (and
|
||||
# str() on every chained exception) and reads error.__traceback__ through the
|
||||
# instance's own __getattribute__, so an exception whose __str__ or
|
||||
# __getattribute__ blocks kept the process alive forever -- the fail-closed
|
||||
# path hung instead of dying (found by the blocking-exception transport tests).
|
||||
_EXC_ARGS = BaseException.args.__get__
|
||||
_EXC_TRACEBACK = BaseException.__traceback__.__get__
|
||||
_EXC_CAUSE = BaseException.__cause__.__get__
|
||||
_EXC_CONTEXT = BaseException.__context__.__get__
|
||||
_EXC_SUPPRESS_CONTEXT = BaseException.__suppress_context__.__get__
|
||||
_TYPE_QUALNAME = type.__dict__["__qualname__"].__get__
|
||||
_TYPE_MODULE = type.__dict__["__module__"].__get__
|
||||
_MAX_CHAIN = 16
|
||||
_MAX_ARG_CHARS = 1024
|
||||
|
||||
# Write end of the per-worker fault pipe (WorkerProc.make_worker_process); the
|
||||
# executor's worker monitor waits on the read end next to the process
|
||||
# sentinels. None outside a worker process (tests, the offload child).
|
||||
_FATAL_NOTIFY_FD: int | None = None
|
||||
|
||||
|
||||
def set_fatal_notify_fd(fd: int | None) -> None:
|
||||
"""Register the fault pipe worker_main received from the executor."""
|
||||
global _FATAL_NOTIFY_FD
|
||||
_FATAL_NOTIFY_FD = fd
|
||||
|
||||
|
||||
def notify_parent_fatal() -> None:
|
||||
"""Tell the executor this worker is dying, before the process is gone.
|
||||
|
||||
The executor's only other signal is the process sentinel, which the kernel
|
||||
closes when the LAST task of the worker leaves -- after the address-space
|
||||
teardown, not at os._exit. On a rank whose GPU work is paired with a live
|
||||
peer that teardown can wait for the peer's exit (in a rank-1 fault test, a
|
||||
fail-closed fault on PP0_TP1 left its leader Z and unreapable for 300 s
|
||||
while rank 0 waited on it in turn; the engine hung on the response
|
||||
queue). One byte here wakes the monitor at once so the failure ladder ends
|
||||
the peers, which is what frees the teardown. Nonblocking, never raises:
|
||||
the caller os._exits right after, whatever happens here.
|
||||
"""
|
||||
fd = _FATAL_NOTIFY_FD
|
||||
if fd is None:
|
||||
return
|
||||
try:
|
||||
os.set_blocking(fd, False)
|
||||
os.write(fd, b"F")
|
||||
except BaseException:
|
||||
pass
|
||||
|
||||
|
||||
def _safe_type_label(cls: type) -> str:
|
||||
"""``module.qualname`` of ``cls`` without running class-defined code."""
|
||||
try:
|
||||
qualname = _TYPE_QUALNAME(cls)
|
||||
module = _TYPE_MODULE(cls)
|
||||
except BaseException:
|
||||
return "<unknown type>"
|
||||
if type(qualname) is not str:
|
||||
qualname = "<unknown type>"
|
||||
if type(module) is str and module != "builtins":
|
||||
return f"{module}.{qualname}"
|
||||
return qualname
|
||||
|
||||
|
||||
def _safe_exception_line(error: BaseException) -> str:
|
||||
"""Type label plus the exact-``str``/``int`` args; nothing else is shown."""
|
||||
label = _safe_type_label(type(error))
|
||||
args = _EXC_ARGS(error)
|
||||
if type(args) is not tuple or len(args) == 0:
|
||||
return label
|
||||
parts = []
|
||||
for arg in args[:4]:
|
||||
if type(arg) is str:
|
||||
parts.append(arg[:_MAX_ARG_CHARS])
|
||||
elif type(arg) is int and -(10**18) < arg < 10**18:
|
||||
parts.append(str(arg))
|
||||
else:
|
||||
parts.append(f"<{_safe_type_label(type(arg))}>")
|
||||
if len(args) > 4:
|
||||
parts.append("...")
|
||||
if len(args) == 1:
|
||||
return f"{label}: {parts[0]}"
|
||||
return f"{label}: ({', '.join(parts)})"
|
||||
|
||||
|
||||
def _safe_frames(error: BaseException) -> list[str]:
|
||||
lines = []
|
||||
tb = _EXC_TRACEBACK(error)
|
||||
while type(tb) is TracebackType:
|
||||
code = tb.tb_frame.f_code
|
||||
filename = code.co_filename
|
||||
name = code.co_qualname
|
||||
lineno = tb.tb_lineno
|
||||
lines.append(
|
||||
f' File "{filename if type(filename) is str else "?"}", '
|
||||
f"line {lineno if type(lineno) is int else '?'}, "
|
||||
f"in {name if type(name) is str else '?'}\n"
|
||||
)
|
||||
tb = tb.tb_next
|
||||
return lines
|
||||
|
||||
|
||||
def _safe_format_exception(error: BaseException) -> str:
|
||||
"""A traceback.format_exception look-alike that runs no exception code.
|
||||
|
||||
Frames come from the traceback objects (file, line, function; no source
|
||||
lines, which would mean file I/O on the dying path); the message is the
|
||||
exception's exact-``str`` args, never ``str(error)``. Chains follow
|
||||
``__cause__`` / ``__context__`` like the interpreter, bounded and
|
||||
cycle-safe.
|
||||
"""
|
||||
chain: list[tuple[BaseException, str]] = []
|
||||
seen: set[int] = set()
|
||||
current: BaseException | None = error
|
||||
link = ""
|
||||
while current is not None and len(chain) < _MAX_CHAIN:
|
||||
if id(current) in seen:
|
||||
break
|
||||
seen.add(id(current))
|
||||
chain.append((current, link))
|
||||
cause = _EXC_CAUSE(current)
|
||||
if cause is not None:
|
||||
current = cause
|
||||
link = (
|
||||
"\nThe above exception was the direct cause of the "
|
||||
"following exception:\n\n"
|
||||
)
|
||||
elif _EXC_SUPPRESS_CONTEXT(current) is not True:
|
||||
current = _EXC_CONTEXT(current)
|
||||
link = (
|
||||
"\nDuring handling of the above exception, another "
|
||||
"exception occurred:\n\n"
|
||||
)
|
||||
else:
|
||||
current = None
|
||||
out: list[str] = []
|
||||
# Innermost first, as the interpreter prints; each outer exception's link
|
||||
# text sits between it and the exception it chains to.
|
||||
for index in range(len(chain) - 1, -1, -1):
|
||||
exc, _ = chain[index]
|
||||
out.append("Traceback (most recent call last):\n")
|
||||
out.extend(_safe_frames(exc))
|
||||
out.append(_safe_exception_line(exc) + "\n")
|
||||
if index > 0:
|
||||
out.append(chain[index][1])
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def ple_fail_closed(
|
||||
*,
|
||||
where: str,
|
||||
rank: int,
|
||||
layer: str,
|
||||
sequence: int,
|
||||
error: BaseException,
|
||||
) -> NoReturn:
|
||||
"""Emit one best-effort nonblocking diagnostic and terminate immediately."""
|
||||
try:
|
||||
os.set_blocking(2, False)
|
||||
error_type = type(error)
|
||||
error_name = "Exception"
|
||||
trusted_error = False
|
||||
for trusted_type, trusted_name in _TRUSTED_ERROR_LABELS:
|
||||
if error_type is trusted_type:
|
||||
error_name = trusted_name
|
||||
trusted_error = True
|
||||
break
|
||||
|
||||
safe_where = where[:128] if type(where) is str else "unknown"
|
||||
safe_layer = layer[:256] if type(layer) is str else "unknown"
|
||||
safe_rank = rank if type(rank) is int else -1
|
||||
safe_sequence = sequence if type(sequence) is int else -1
|
||||
detail_suffix = ""
|
||||
if trusted_error:
|
||||
args = BaseException.args.__get__(error, error_type)
|
||||
if type(args) is tuple and len(args) == 1 and type(args[0]) is str:
|
||||
detail = args[0][:512].replace("\n", "\\n")
|
||||
detail_suffix = f":{detail}"
|
||||
message = (
|
||||
f"PLE fatal: where={safe_where} rank={safe_rank} layer={safe_layer} "
|
||||
f"sequence={safe_sequence} error={error_name}{detail_suffix} "
|
||||
f"pid={os.getpid()} "
|
||||
f"monotonic={time.monotonic():.9f} wall={time.time():.9f}\n"
|
||||
)
|
||||
os.write(2, message.encode("utf-8", "backslashreplace")[:2048])
|
||||
except BaseException:
|
||||
pass
|
||||
# THE TRACEBACK, unconditionally, in its own try so it cannot cost us the
|
||||
# line above.
|
||||
#
|
||||
# This path os._exit(1)s, which runs no handlers and flushes nothing, so
|
||||
# whatever is not written here is gone. The line above names the TYPE only
|
||||
# when the type is one of _TRUSTED_ERROR_LABELS and flattens everything
|
||||
# else to the literal string "Exception" -- and the flattened set includes
|
||||
# the two that matter most here: torch.AcceleratorError (a RuntimeError
|
||||
# SUBCLASS, so the `is` comparison misses it) and Triton CompilationError.
|
||||
# An earlier boot died exactly that way: both PP0 ranks
|
||||
# reported "error=Exception" and there was no traceback anywhere in the
|
||||
# log, in any rank, so the cause could not be named at all and the next
|
||||
# step was a re-run rather than a fix.
|
||||
#
|
||||
# A fail-closed path is where evidence matters most, not least: it fires
|
||||
# once, it kills the process, and it is the only witness. Printing the type
|
||||
# of a wrapped exception without the frames that raised it is not a
|
||||
# diagnostic.
|
||||
try:
|
||||
# Three-arg form: it does not depend on being inside an active except
|
||||
# block, and this function is called from one frame further out than
|
||||
# the raise.
|
||||
# _safe_format_exception, not traceback.format_exception: see the note
|
||||
# above _EXC_ARGS -- the stdlib formatter calls str(error), which is
|
||||
# exception-defined code and can block the dying process forever.
|
||||
tb = _safe_format_exception(error)
|
||||
raw = tb.encode("utf-8", "backslashreplace")
|
||||
# Truncate from the END, not the start. format_exception puts the
|
||||
# frames first and the exception TYPE AND MESSAGE last, so a head-side
|
||||
# cut throws away the one line that names what went wrong and keeps the
|
||||
# import-time frames nobody needs. A long chained traceback is exactly
|
||||
# the case where that matters.
|
||||
clipped = len(raw) > 8192
|
||||
raw = raw[-8192:]
|
||||
os.write(
|
||||
2,
|
||||
b"PLE fatal traceback (tail) follows:\n"
|
||||
if clipped
|
||||
else b"PLE fatal traceback follows:\n",
|
||||
)
|
||||
os.write(2, raw)
|
||||
if not raw.endswith(b"\n"):
|
||||
os.write(2, b"\n")
|
||||
except BaseException:
|
||||
# Never silent HERE of all places: without this the reader cannot tell
|
||||
# "the wrapper predates the traceback fix" from "the traceback itself
|
||||
# could not be produced", and those call for different next steps.
|
||||
# Still guarded, because os._exit below must run whatever happens: an
|
||||
# escape from this handler would leave the worker alive in the state
|
||||
# the fail-closed path exists to end.
|
||||
try: # noqa: SIM105
|
||||
os.write(2, b"PLE fatal: traceback unavailable\n")
|
||||
except BaseException:
|
||||
# SIM105 declined: an earlier revision made this helper call no
|
||||
# exception-defined code on the dying path. contextlib.suppress
|
||||
# runs a Python __exit__ between here and os._exit below; a bare
|
||||
# except keeps that window empty.
|
||||
pass
|
||||
notify_parent_fatal()
|
||||
os._exit(1)
|
||||
|
||||
|
||||
def fail_ple_execute_model(func: _F) -> _F:
|
||||
"""Fail a non-shutdown worker-step escape only on PLE-owning ranks.
|
||||
|
||||
This includes ``sample_tokens``, the second half of the split execute-model
|
||||
step. A rank without a PLE connector rethrows the original exception.
|
||||
"""
|
||||
|
||||
@functools.wraps(func)
|
||||
def wrapped(self: Any, *args: Any, **kwargs: Any) -> Any:
|
||||
try:
|
||||
return func(self, *args, **kwargs)
|
||||
except SystemExit:
|
||||
# WorkerProc's signal handler uses SystemExit for clean shutdown.
|
||||
raise
|
||||
except BaseException as error:
|
||||
connector = getattr(self, "_ple_offload_connector", None)
|
||||
if connector is None:
|
||||
model_runner = getattr(self, "model_runner", None)
|
||||
connector = getattr(model_runner, "_ple_offload_connector", None)
|
||||
if connector is None:
|
||||
raise
|
||||
connector._fail_closed(func.__name__, error)
|
||||
|
||||
return wrapped # type: ignore[return-value]
|
||||
@@ -0,0 +1,305 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Independent progress deadline for PLE transactions."""
|
||||
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.v1.ple_offload.fatal import ple_fail_closed
|
||||
|
||||
_DEFAULT_SERVING_DEADLINE_S = 300.0
|
||||
_DEFAULT_STARTUP_DEADLINE_S = 1800.0
|
||||
_MAX_PROGRESS_DEADLINE_S = 86400.0
|
||||
_POLL_INTERVAL_S = 0.1
|
||||
|
||||
|
||||
def _validate_deadline(value: float, name: str) -> float:
|
||||
value = float(value)
|
||||
if not math.isfinite(value) or value <= 0 or value > _MAX_PROGRESS_DEADLINE_S:
|
||||
raise ValueError(
|
||||
f"{name} must be finite, greater than zero, and no greater than "
|
||||
f"{_MAX_PROGRESS_DEADLINE_S:g} seconds"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def _environment_deadline(name: str, default: float) -> float:
|
||||
value = float(os.environ.get(name, str(default)))
|
||||
return _validate_deadline(value, name)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Submission:
|
||||
sequence: int
|
||||
started: float
|
||||
startup: bool
|
||||
num_tokens: int
|
||||
deadline: float
|
||||
depth: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _PendingCompletion:
|
||||
sequence: int
|
||||
started: float
|
||||
startup: bool
|
||||
num_tokens: int
|
||||
deadline: float
|
||||
event: Any
|
||||
|
||||
|
||||
class PleProgressMonitor:
|
||||
"""Bound host submission and the resulting ordered GPU completions."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
rank: int,
|
||||
layer_names: tuple[str, ...],
|
||||
serving_deadline_s: float,
|
||||
startup_deadline_s: float,
|
||||
) -> None:
|
||||
serving_deadline_s = _validate_deadline(
|
||||
serving_deadline_s, "serving_deadline_s"
|
||||
)
|
||||
startup_deadline_s = _validate_deadline(
|
||||
startup_deadline_s, "startup_deadline_s"
|
||||
)
|
||||
self._rank = rank
|
||||
self._layer = ",".join(layer_names)
|
||||
self._serving_deadline_s = serving_deadline_s
|
||||
self._startup_deadline_s = startup_deadline_s
|
||||
self._condition = threading.Condition()
|
||||
self._submission: _Submission | None = None
|
||||
self._pending: deque[_PendingCompletion] = deque()
|
||||
self._sequence = 0
|
||||
self._stopping = False
|
||||
self._thread = threading.Thread(
|
||||
target=self._run,
|
||||
name=f"ple-progress-rank{rank}",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
@classmethod
|
||||
def from_environment(
|
||||
cls, *, rank: int, layer_names: tuple[str, ...]
|
||||
) -> "PleProgressMonitor":
|
||||
return cls(
|
||||
rank=rank,
|
||||
layer_names=layer_names,
|
||||
serving_deadline_s=_environment_deadline(
|
||||
"VLLM_PLE_PROGRESS_DEADLINE_S",
|
||||
_DEFAULT_SERVING_DEADLINE_S,
|
||||
),
|
||||
startup_deadline_s=_environment_deadline(
|
||||
"VLLM_PLE_STARTUP_DEADLINE_S",
|
||||
_DEFAULT_STARTUP_DEADLINE_S,
|
||||
),
|
||||
)
|
||||
|
||||
@property
|
||||
def sequence(self) -> int:
|
||||
with self._condition:
|
||||
return self._sequence
|
||||
|
||||
@property
|
||||
def has_pending_completions(self) -> bool:
|
||||
with self._condition:
|
||||
return bool(self._pending)
|
||||
|
||||
def describe_submission(self, sequence: int) -> str:
|
||||
"""Return the armed transaction descriptor for the active sequence."""
|
||||
with self._condition:
|
||||
active = self._submission
|
||||
if active is None or active.sequence != sequence:
|
||||
raise RuntimeError("PLE fault correlation has no active submission")
|
||||
return f"sequence:{active.sequence}@begin:{active.started:.9f}"
|
||||
|
||||
def completion_is_pending(self, sequence: int) -> bool:
|
||||
"""Whether this serving sequence has a retained, incomplete event.
|
||||
|
||||
This method is called only behind the armed fault guard. An event that
|
||||
has completed on the GPU but has not yet been dequeued is not counted.
|
||||
"""
|
||||
with self._condition:
|
||||
pending = next(
|
||||
(
|
||||
item
|
||||
for item in self._pending
|
||||
if item.sequence == sequence and not item.startup
|
||||
),
|
||||
None,
|
||||
)
|
||||
if pending is None:
|
||||
return False
|
||||
return not bool(pending.event.query())
|
||||
|
||||
def begin(self, *, startup: bool, num_tokens: int) -> int:
|
||||
now = time.monotonic()
|
||||
budget = self._startup_deadline_s if startup else self._serving_deadline_s
|
||||
with self._condition:
|
||||
active = self._submission
|
||||
if active is not None:
|
||||
if not startup or not active.startup:
|
||||
raise RuntimeError(
|
||||
"Overlapping PLE host submissions violate the serialized "
|
||||
"single-slot protocol"
|
||||
)
|
||||
sequence = active.sequence
|
||||
self._submission = _Submission(
|
||||
sequence=sequence,
|
||||
started=active.started,
|
||||
startup=True,
|
||||
num_tokens=num_tokens,
|
||||
deadline=now + budget,
|
||||
depth=active.depth + 1,
|
||||
)
|
||||
else:
|
||||
self._sequence += 1
|
||||
sequence = self._sequence
|
||||
self._submission = _Submission(
|
||||
sequence=sequence,
|
||||
started=now,
|
||||
startup=startup,
|
||||
num_tokens=num_tokens,
|
||||
deadline=now + budget,
|
||||
depth=1,
|
||||
)
|
||||
self._condition.notify_all()
|
||||
return sequence
|
||||
|
||||
def track_completion(self, event: Any) -> None:
|
||||
"""Retain a release event until the monitor observes GPU completion."""
|
||||
if not callable(getattr(event, "query", None)):
|
||||
raise TypeError("PLE completion event must provide query()")
|
||||
with self._condition:
|
||||
active = self._submission
|
||||
if active is None:
|
||||
raise RuntimeError("PLE completion has no matching submission")
|
||||
self._pending.append(
|
||||
_PendingCompletion(
|
||||
sequence=active.sequence,
|
||||
started=active.started,
|
||||
startup=active.startup,
|
||||
num_tokens=active.num_tokens,
|
||||
deadline=active.deadline,
|
||||
event=event,
|
||||
)
|
||||
)
|
||||
self._condition.notify_all()
|
||||
|
||||
def complete(self) -> None:
|
||||
with self._condition:
|
||||
active = self._submission
|
||||
if active is None or active.depth <= 0:
|
||||
raise RuntimeError("PLE release has no matching transaction")
|
||||
if active.depth == 1:
|
||||
self._submission = None
|
||||
else:
|
||||
self._submission = _Submission(
|
||||
sequence=active.sequence,
|
||||
started=active.started,
|
||||
startup=active.startup,
|
||||
num_tokens=active.num_tokens,
|
||||
deadline=active.deadline,
|
||||
depth=active.depth - 1,
|
||||
)
|
||||
self._condition.notify_all()
|
||||
|
||||
def close(self) -> None:
|
||||
with self._condition:
|
||||
self._stopping = True
|
||||
self._condition.notify_all()
|
||||
self._thread.join(timeout=1)
|
||||
|
||||
def _run(self) -> None:
|
||||
try:
|
||||
self._monitor()
|
||||
except BaseException as error:
|
||||
ple_fail_closed(
|
||||
where="progress-monitor-thread",
|
||||
rank=self._rank,
|
||||
layer=self._layer,
|
||||
sequence=self._sequence,
|
||||
error=error,
|
||||
)
|
||||
|
||||
def _monitor(self) -> None:
|
||||
while True:
|
||||
with self._condition:
|
||||
if self._stopping:
|
||||
return
|
||||
submission = self._submission
|
||||
pending = self._pending[0] if self._pending else None
|
||||
if submission is None and pending is None:
|
||||
self._condition.wait(timeout=_POLL_INTERVAL_S)
|
||||
continue
|
||||
|
||||
# CUDA completion is observed only here. Never synchronize and never
|
||||
# use the host flag as evidence that the queued reset completed.
|
||||
if pending is not None and pending.event.query():
|
||||
with self._condition:
|
||||
if self._pending and self._pending[0] is pending:
|
||||
self._pending.popleft()
|
||||
self._condition.notify_all()
|
||||
continue
|
||||
|
||||
now = time.monotonic()
|
||||
if pending is not None and now >= pending.deadline:
|
||||
with self._condition:
|
||||
if not self._pending or self._pending[0] is not pending:
|
||||
continue
|
||||
mode = "startup" if pending.startup else "serving"
|
||||
ple_fail_closed(
|
||||
where=(
|
||||
f"progress-deadline:{mode}:gpu-completion:"
|
||||
f"tokens={pending.num_tokens}"
|
||||
),
|
||||
rank=self._rank,
|
||||
layer=self._layer,
|
||||
sequence=pending.sequence,
|
||||
error=TimeoutError("PLE GPU completion made no bounded progress"),
|
||||
)
|
||||
|
||||
if submission is not None and now >= submission.deadline:
|
||||
with self._condition:
|
||||
if self._submission is not submission:
|
||||
continue
|
||||
mode = "startup" if submission.startup else "serving"
|
||||
ple_fail_closed(
|
||||
where=(
|
||||
f"progress-deadline:{mode}:host-submission:"
|
||||
f"tokens={submission.num_tokens}"
|
||||
),
|
||||
rank=self._rank,
|
||||
layer=self._layer,
|
||||
sequence=submission.sequence,
|
||||
error=TimeoutError("PLE host submission made no bounded progress"),
|
||||
)
|
||||
|
||||
deadlines = []
|
||||
if pending is not None:
|
||||
deadlines.append(pending.deadline)
|
||||
if submission is not None:
|
||||
deadlines.append(submission.deadline)
|
||||
timeout = _POLL_INTERVAL_S
|
||||
if deadlines:
|
||||
timeout = min(
|
||||
timeout,
|
||||
max(0.0, min(deadlines) - time.monotonic()),
|
||||
)
|
||||
with self._condition:
|
||||
if self._stopping:
|
||||
return
|
||||
if (
|
||||
self._submission is not submission
|
||||
or (self._pending[0] if self._pending else None) is not pending
|
||||
):
|
||||
continue
|
||||
self._condition.wait(timeout=timeout)
|
||||
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""IPC message definitions for PLE CPU offload (host-mapped pull design).
|
||||
|
||||
The offload process owns no CUDA context. Every GPU worker owns, per PLE
|
||||
layer, a shared host output buffer and a shared host int32 flag, both
|
||||
page-locked and device-mapped in the worker's context. The offload process
|
||||
waits for ``flag == 0``, writes the layer result into the host buffer and
|
||||
stores ``flag = 1`` with plain CPU stores; the GPU's captured forward waits on
|
||||
the host flag, DMAs host -> GPU on the model stream, and ``release_outputs``
|
||||
resets the flag after the forward.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# IPC message dataclasses
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class PleOffloadRegistration:
|
||||
"""Sent once from each GPU worker during offload setup."""
|
||||
|
||||
worker_id: int
|
||||
tp_rank: int
|
||||
dp_rank: int
|
||||
# Shared CPU tensors per PLE layer, filled by the offload process.
|
||||
host_output_buffers: dict[str, torch.Tensor]
|
||||
# Shared CPU int32 tensors per PLE layer; element 0 is the ready flag.
|
||||
host_flag_tensors: dict[str, torch.Tensor]
|
||||
# CPU tensors are allocated in shared memory and registered once.
|
||||
input_ids_buf: torch.Tensor
|
||||
query_start_loc_buf: torch.Tensor
|
||||
ngram_context_buf: torch.Tensor | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PleOffloadRequest:
|
||||
"""Sent by each DP rank's TP rank zero at every inference step."""
|
||||
|
||||
dp_rank: int
|
||||
num_tokens: int
|
||||
num_reqs: int
|
||||
|
||||
|
||||
_PLE_OFFLOAD_REQUEST_DECODER = msgspec.msgpack.Decoder(PleOffloadRequest)
|
||||
@@ -0,0 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Compatibility exports for the top-level test-only fault singleton."""
|
||||
|
||||
from vllm_ple_test_fault import (
|
||||
_FAULT as _FAULT,
|
||||
)
|
||||
from vllm_ple_test_fault import ( # noqa: F401
|
||||
FaultSpec as FaultSpec,
|
||||
)
|
||||
from vllm_ple_test_fault import (
|
||||
_parse_fault_spec as _parse_fault_spec,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,266 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Per-rank launch-identity log line (log-only; changes no behaviour).
|
||||
|
||||
After model loading -- and after the PLE offload placeholders are armed -- every
|
||||
GPU worker rank logs one line per model role it holds::
|
||||
|
||||
e2-identity[PP<p>_TP<t>]: ple_backend=<class>@<transport> ple_method=<class>
|
||||
ple_weight_dtype=<dtype> breakable=<ON|OFF> compile_mode=<mode>
|
||||
cudagraph_mode=<mode> role=<target|draft>
|
||||
|
||||
(one line in the log). ``role=target`` is logged after model loading;
|
||||
``role=draft`` is logged only where a draft model is loaded (the last PP
|
||||
stage), once the drafter's graph modes are resolved (after the KV cache is
|
||||
initialised -- see below). Every value is read from the live objects:
|
||||
|
||||
* ``ple_backend``: the class of each PLE layer's ``ple_embedding`` module, and
|
||||
the transport that serves its rows on this rank, read from the placeholder's
|
||||
armed state -- ``host-mapped-pull`` (``_is_cpu_offloaded`` with a host output
|
||||
pointer), ``ipc-push`` (offloaded, no host pointer) or ``local:<device>`` (the
|
||||
rank owns the table).
|
||||
* ``ple_method``: the embedding's live ``quant_method``, or, on an offloaded
|
||||
placeholder, the ``_offload_quant_method`` the rank dequantizes with.
|
||||
* ``ple_weight_dtype``: the table's dtype where the rank owns it; on an
|
||||
offloaded placeholder, the dtype of the armed transfer buffer, into which the
|
||||
offload process copies the table's rows unchanged
|
||||
(``get_offload_output_dtype``).
|
||||
* ``breakable``: ``is_breakable_cudagraph_enabled()``, the switch the capture
|
||||
path itself reads.
|
||||
* ``compile_mode`` / ``cudagraph_mode`` for ``role=target``: the engine's
|
||||
``CompilationConfig`` (config level, i.e. before KV-cache-time capture-mode
|
||||
resolution).
|
||||
* ``compile_mode`` / ``cudagraph_mode`` for ``role=draft``: the DRAFTER's own
|
||||
resolved state, not the config. The drafter is built with the target's
|
||||
VllmConfig and there is one ``CompilationConfig`` in the engine, so the
|
||||
config fields name the target's profile (the trap an earlier commit fixed for
|
||||
the draft guard's architectures). What the drafter actually runs is:
|
||||
|
||||
- ``compile_mode``: the mode of the draft model's live
|
||||
``@support_torch_compile`` wrapper(s) (``do_not_compile`` false -- the
|
||||
property the capture path tests, ``gpu/cudagraph_utils.py``
|
||||
``has_compiled_submodule``); ``NONE`` when no wrapper is live.
|
||||
- ``cudagraph_mode``: the mode each speculator cudagraph manager holds after
|
||||
``init_cudagraph_manager`` (V2 runner): ``prefill:<m>,decode:<m>`` for the
|
||||
autoregressive / MTP speculator, the bare mode for a single-manager
|
||||
speculator; for a V1 drafter, its dispatcher's mode. These are the values
|
||||
``e2-guard[runtime]`` checks before capture. ``unresolved`` if a manager
|
||||
does not exist yet.
|
||||
|
||||
Hence the draft line is logged from ``Worker.initialize_from_config`` after
|
||||
``initialize_kv_cache``, not at load time.
|
||||
|
||||
A role with no PLE layer reports ``none`` for the three PLE fields; several
|
||||
distinct values on one rank are joined with ``,``.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from torch import nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_NONE = "none"
|
||||
|
||||
|
||||
def _join(values: set[str]) -> str:
|
||||
return ",".join(sorted(values)) if values else _NONE
|
||||
|
||||
|
||||
def _enum_name(value: Any) -> str:
|
||||
if value is None:
|
||||
return _NONE
|
||||
return str(getattr(value, "name", value))
|
||||
|
||||
|
||||
def _ple_embeddings(model: nn.Module | None) -> list[nn.Module]:
|
||||
"""The ``ple_embedding`` module of every PLE layer in ``model``."""
|
||||
if model is None:
|
||||
return []
|
||||
found = []
|
||||
for module in model.modules():
|
||||
embedding = module.__dict__.get("_modules", {}).get("ple_embedding")
|
||||
if isinstance(embedding, nn.Module):
|
||||
found.append(embedding)
|
||||
return found
|
||||
|
||||
|
||||
def _transport(embedding: nn.Module) -> str:
|
||||
if getattr(embedding, "_is_cpu_offloaded", False):
|
||||
if getattr(embedding, "_host_output_ptr", None) is not None:
|
||||
return "host-mapped-pull"
|
||||
return "ipc-push"
|
||||
weight = getattr(getattr(embedding, "ngram_embedding", None), "weight", None)
|
||||
if weight is not None:
|
||||
return f"local:{weight.device.type}"
|
||||
return "local:unknown"
|
||||
|
||||
|
||||
def _method(embedding: nn.Module) -> str:
|
||||
table = getattr(embedding, "ngram_embedding", None)
|
||||
method = getattr(table, "quant_method", None)
|
||||
if method is None:
|
||||
method = getattr(embedding, "_offload_quant_method", None)
|
||||
return type(method).__name__ if method is not None else _NONE
|
||||
|
||||
|
||||
def _weight_dtype(embedding: nn.Module) -> str:
|
||||
weight = getattr(getattr(embedding, "ngram_embedding", None), "weight", None)
|
||||
if weight is not None:
|
||||
return str(weight.dtype)
|
||||
if getattr(embedding, "_is_cpu_offloaded", False):
|
||||
buffer = getattr(embedding, "_gpu_output_buffer", None)
|
||||
if buffer is not None:
|
||||
return str(buffer.dtype)
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _rank_tag() -> str:
|
||||
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
|
||||
|
||||
return f"PP{get_pp_group().rank_in_group}_TP{get_tp_group().rank_in_group}"
|
||||
|
||||
|
||||
def _draft(model_runner: Any) -> tuple[nn.Module | None, Any]:
|
||||
"""The loaded draft model and its holder (V2 speculator or V1 drafter)."""
|
||||
for name in ("speculator", "drafter"):
|
||||
holder = getattr(model_runner, name, None)
|
||||
model = getattr(holder, "model", None)
|
||||
if isinstance(model, nn.Module):
|
||||
return model, holder
|
||||
return None, None
|
||||
|
||||
|
||||
#: Written where a draft graph mode has no resolved source yet.
|
||||
UNRESOLVED = "unresolved"
|
||||
#: Speculator cudagraph-manager attributes (the set ``e2-guard[runtime]``
|
||||
#: reads), with the phase label each carries on the draft line.
|
||||
_DRAFT_MANAGER_PHASES = (
|
||||
("prefill_cudagraph_manager", "prefill"),
|
||||
("decode_cudagraph_manager", "decode"),
|
||||
("cudagraph_manager", None),
|
||||
)
|
||||
|
||||
|
||||
def _draft_compile_mode(model: nn.Module) -> str:
|
||||
"""Compile mode of the draft model's live ``@support_torch_compile`` wrappers.
|
||||
|
||||
``NONE`` when no wrapper is live: the drafter then runs eager, whatever
|
||||
the shared ``CompilationConfig.mode`` says.
|
||||
"""
|
||||
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
|
||||
|
||||
modes = {
|
||||
_enum_name(getattr(getattr(m, "compilation_config", None), "mode", None))
|
||||
for m in model.modules()
|
||||
if isinstance(m, TorchCompileWithNoGuardsWrapper)
|
||||
and not getattr(m, "do_not_compile", True)
|
||||
}
|
||||
return _join(modes) if modes else "NONE"
|
||||
|
||||
|
||||
def _draft_cudagraph_mode(holder: Any) -> str:
|
||||
"""The drafter's resolved cudagraph mode(s), read from its managers."""
|
||||
parts = []
|
||||
for attr, phase in _DRAFT_MANAGER_PHASES:
|
||||
if not hasattr(holder, attr):
|
||||
continue
|
||||
manager = getattr(holder, attr)
|
||||
mode = (
|
||||
UNRESOLVED
|
||||
if manager is None
|
||||
else _enum_name(getattr(manager, "cudagraph_mode", None))
|
||||
)
|
||||
parts.append(mode if phase is None else f"{phase}:{mode}")
|
||||
if parts:
|
||||
return ",".join(parts)
|
||||
# V1 drafter: one dispatcher, keyed in _check_and_update_cudagraph_mode.
|
||||
dispatcher = getattr(holder, "cudagraph_dispatcher", None)
|
||||
if dispatcher is not None and getattr(dispatcher, "keys_initialized", False):
|
||||
return _enum_name(getattr(dispatcher, "cudagraph_mode", None))
|
||||
return UNRESOLVED
|
||||
|
||||
|
||||
def identity_line(
|
||||
tag: str,
|
||||
model: nn.Module | None,
|
||||
vllm_config: Any,
|
||||
role: str,
|
||||
*,
|
||||
compile_mode: str | None = None,
|
||||
cudagraph_mode: str | None = None,
|
||||
) -> str:
|
||||
"""One identity line. ``compile_mode`` / ``cudagraph_mode`` override the
|
||||
config-level values (the draft role passes its resolved state)."""
|
||||
from vllm.compilation.breakable_cudagraph import is_breakable_cudagraph_enabled
|
||||
|
||||
embeddings = _ple_embeddings(model)
|
||||
backends = {f"{type(e).__name__}@{_transport(e)}" for e in embeddings}
|
||||
compilation = getattr(vllm_config, "compilation_config", None)
|
||||
if compile_mode is None:
|
||||
compile_mode = _enum_name(getattr(compilation, "mode", None))
|
||||
if cudagraph_mode is None:
|
||||
cudagraph_mode = _enum_name(getattr(compilation, "cudagraph_mode", None))
|
||||
return (
|
||||
f"e2-identity[{tag}]: "
|
||||
f"ple_backend={_join(backends)} "
|
||||
f"ple_method={_join({_method(e) for e in embeddings})} "
|
||||
f"ple_weight_dtype={_join({_weight_dtype(e) for e in embeddings})} "
|
||||
f"breakable={'ON' if is_breakable_cudagraph_enabled() else 'OFF'} "
|
||||
f"compile_mode={compile_mode} "
|
||||
f"cudagraph_mode={cudagraph_mode} "
|
||||
f"role={role}"
|
||||
)
|
||||
|
||||
|
||||
def draft_identity_line(tag: str, draft_model: nn.Module, holder: Any) -> str:
|
||||
"""The ``role=draft`` line: PLE fields from the draft modules, graph fields
|
||||
from the drafter's resolved state (never the shared CompilationConfig)."""
|
||||
return identity_line(
|
||||
tag,
|
||||
draft_model,
|
||||
getattr(holder, "vllm_config", None),
|
||||
"draft",
|
||||
compile_mode=_draft_compile_mode(draft_model),
|
||||
cudagraph_mode=_draft_cudagraph_mode(holder),
|
||||
)
|
||||
|
||||
|
||||
def log_launch_identity(model_runner: Any, vllm_config: Any) -> None:
|
||||
"""Log this rank's ``role=target`` line after model load. Never raises.
|
||||
|
||||
The ``role=draft`` line comes later, from :func:`log_draft_identity`, once
|
||||
the drafter's graph modes exist.
|
||||
"""
|
||||
tag = "PP?_TP?"
|
||||
try:
|
||||
tag = _rank_tag()
|
||||
logger.info(
|
||||
"%s", identity_line(tag, model_runner.get_model(), vllm_config, "target")
|
||||
)
|
||||
except Exception as error: # log-only: identity must never break a boot
|
||||
logger.warning(
|
||||
"e2-identity[%s]: unavailable (%s: %s)", tag, type(error).__name__, error
|
||||
)
|
||||
|
||||
|
||||
def log_draft_identity(model_runner: Any) -> None:
|
||||
"""Log this rank's ``role=draft`` line, if it holds a drafter. Never raises.
|
||||
|
||||
Call after ``initialize_kv_cache``: that is where the V2 runner resolves
|
||||
the target's mode and ``init_cudagraph_manager`` derives the drafter's
|
||||
(and where the V1 runner keys the drafter's dispatcher).
|
||||
"""
|
||||
tag = "PP?_TP?"
|
||||
try:
|
||||
draft_model, holder = _draft(model_runner)
|
||||
if draft_model is None:
|
||||
return
|
||||
tag = _rank_tag()
|
||||
logger.info("%s", draft_identity_line(tag, draft_model, holder))
|
||||
except Exception as error: # log-only: identity must never break a boot
|
||||
logger.warning(
|
||||
"e2-identity[%s]: unavailable (%s: %s)", tag, type(error).__name__, error
|
||||
)
|
||||
@@ -0,0 +1,200 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Debug-only allocator telemetry for E2, BOUNDARY-ONLY. Off unless
|
||||
VLLM_E2_MEMSTATS=<ctl poll seconds> and VLLM_E2_MEMSTATS_DIR=<dir>.
|
||||
|
||||
WHY THIS IS NOT A SAMPLER ANY MORE. An earlier version read
|
||||
torch.cuda.memory_stats() and reset_peak_memory_stats() every VLLM_E2_MEMSTATS
|
||||
seconds. Both take the per-device caching-allocator mutex and the Python
|
||||
bindings do not release the GIL, so the old contract's claim that this thread
|
||||
"takes no lock shared with the model path" was FALSE. Three one-boot arms at a
|
||||
4.1e9 KV pin showed a monotone dose-response: sampling at 4 Hz wedged a stage-0
|
||||
worker during a 131K thinking-on row; at 1.1 Hz it survived that row and wedged during
|
||||
a 260K row; with the thread disabled the run completed every row. Causation
|
||||
is not established at n=1 per arm (a known limit of
|
||||
that capacity probe), but polling has no place
|
||||
inside a measured region, and it was never necessary: PyTorch maintains the
|
||||
allocator high-water marks itself, so reading them once per window is exact.
|
||||
|
||||
WHAT THIS THREAD DOES NOW. Between window boundaries it makes NO CUDA calls and
|
||||
touches NO allocator state: its tick is a plain read of <dir>/window.ctl, which
|
||||
holds "<seq> <name>". The gate driver crosses a boundary only while the server
|
||||
is quiet (no request running). On a new seq the thread performs these
|
||||
boundary-only allocator reads:
|
||||
1. memory_stats() -> the closing window's exact high-water marks
|
||||
and cumulative retry/OOM counters
|
||||
2. reset_peak_memory_stats() -> immediately after the read, with no file I/O
|
||||
in the gap
|
||||
3. mem_get_info() -> device free/total AT THAT QUIESCENT INSTANT
|
||||
4. host_memory_stats() -> pinned-host allocator current values, outside
|
||||
the read-to-reset gap
|
||||
then writes the "close" record for the old window and the "ack" record for the
|
||||
new one.
|
||||
|
||||
WHAT THIS MEASURES.
|
||||
- A row's allocator peak is EXACT for that row's window (not a sampled
|
||||
maximum; it cannot miss a transient).
|
||||
- Device free/total is observed ONLY at quiescent boundaries. No minimum-free
|
||||
figure exists for the inside of a row, and none may be reported.
|
||||
- Retry and OOM counters are cumulative and never reset; per-row deltas come
|
||||
from consecutive close records.
|
||||
- There is NO interval-level peak. A qualifying interval is narrower
|
||||
than its row and is identified only afterwards from the scheduler log, so
|
||||
the peak attributable to it is the ROW-level high-water mark, which
|
||||
over-includes the prefill ramp. Report this loss of resolution
|
||||
explicitly.
|
||||
|
||||
Records (one JSON per line): kind start | close | ack | error; rank, local,
|
||||
dev, pid; window, seq; and for close: allocated_bytes.all.current,
|
||||
reserved_bytes.all.current, alloc_peak_w, res_peak_w, retries, ooms, free,
|
||||
total, run_alloc_peak, run_res_peak, and a host object containing every
|
||||
host_memory_stats() key ending in ".current". Records are written with
|
||||
synchronous file I/O, never inside the read-to-reset gap. Nothing here raises
|
||||
into the worker.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
|
||||
import torch
|
||||
|
||||
_SEG = float(os.environ.get("VLLM_E2_MEMSTATS", "0") or 0)
|
||||
_DIR = os.environ.get("VLLM_E2_MEMSTATS_DIR", "")
|
||||
_started = []
|
||||
VERSION = "e2_memstats/5-boundary-current-host"
|
||||
|
||||
|
||||
def _close_metrics(dev):
|
||||
"""Read one quiescent boundary; kept separate for CPU-only testing."""
|
||||
s = torch.cuda.memory_stats(dev)
|
||||
ap = int(s.get("allocated_bytes.all.peak", 0))
|
||||
rp = int(s.get("reserved_bytes.all.peak", 0))
|
||||
torch.cuda.reset_peak_memory_stats(dev) # no file I/O in the gap
|
||||
free, total = torch.cuda.mem_get_info(dev)
|
||||
host_stats = torch.cuda.host_memory_stats()
|
||||
host = {
|
||||
key: int(value) for key, value in host_stats.items() if key.endswith(".current")
|
||||
}
|
||||
return {
|
||||
"allocated_bytes.all.current": int(s.get("allocated_bytes.all.current", 0)),
|
||||
"reserved_bytes.all.current": int(s.get("reserved_bytes.all.current", 0)),
|
||||
"alloc_peak_w": ap,
|
||||
"res_peak_w": rp,
|
||||
"retries": int(s.get("num_alloc_retries", 0)),
|
||||
"ooms": int(s.get("num_ooms", 0)),
|
||||
"free": int(free),
|
||||
"total": int(total),
|
||||
"host": host,
|
||||
}
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
return _SEG > 0 and bool(_DIR)
|
||||
|
||||
|
||||
def ensure_started(worker, armed_by: str = "compile_or_warm_up_model") -> None:
|
||||
if _started:
|
||||
return
|
||||
dev = (
|
||||
torch.cuda.current_device()
|
||||
) # captured on the worker's own thread: every call below passes it explicitly
|
||||
t = threading.Thread(
|
||||
target=_loop,
|
||||
name="e2-memstats",
|
||||
daemon=True,
|
||||
args=(
|
||||
dev,
|
||||
getattr(worker, "rank", -1),
|
||||
getattr(worker, "local_rank", -1),
|
||||
armed_by,
|
||||
),
|
||||
)
|
||||
_started.append(t)
|
||||
t.start()
|
||||
|
||||
|
||||
def _ctl():
|
||||
try:
|
||||
with open(os.path.join(_DIR, "window.ctl")) as f:
|
||||
seq, name = f.read().split(None, 1)
|
||||
return int(seq), name.strip()
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _loop(dev, rank, local, armed_by):
|
||||
os.makedirs(_DIR, exist_ok=True)
|
||||
# noqa SIM115: the handle lives as long as the telemetry daemon thread
|
||||
# that writes to it; there is no scope to close it in.
|
||||
f = open( # noqa: SIM115
|
||||
os.path.join(_DIR, f"memstats-rank{rank}-pid{os.getpid()}.jsonl"),
|
||||
"a",
|
||||
buffering=1,
|
||||
)
|
||||
base = {"rank": rank, "local": local, "dev": dev, "pid": os.getpid()}
|
||||
|
||||
def w(rec):
|
||||
f.write(json.dumps({**base, **rec}, separators=(",", ":")) + "\n")
|
||||
|
||||
run_ap = run_rp = 0
|
||||
seq, name = _ctl() or (0, "serve")
|
||||
torch.cuda.reset_peak_memory_stats(dev)
|
||||
w(
|
||||
{
|
||||
"kind": "start",
|
||||
"t": round(time.time(), 4),
|
||||
"m": round(time.monotonic(), 4),
|
||||
"armed_by": armed_by,
|
||||
"window": name,
|
||||
"seq": seq,
|
||||
"version": VERSION,
|
||||
"poll_s": _SEG,
|
||||
}
|
||||
)
|
||||
while True:
|
||||
time.sleep(_SEG)
|
||||
try:
|
||||
c = _ctl() # file read only: no CUDA, no allocator lock, no GIL-held call
|
||||
if c is None or c[0] == seq:
|
||||
continue
|
||||
# Boundary. The driver only crosses one while the server is
|
||||
# quiet, so this is the single point at which this thread
|
||||
# touches the allocator at all.
|
||||
metrics = _close_metrics(dev)
|
||||
ap = metrics["alloc_peak_w"]
|
||||
rp = metrics["res_peak_w"]
|
||||
run_ap, run_rp = max(run_ap, ap), max(run_rp, rp)
|
||||
w(
|
||||
{
|
||||
"kind": "close",
|
||||
"t": round(time.time(), 4),
|
||||
"m": round(time.monotonic(), 4),
|
||||
"window": name,
|
||||
"seq": seq,
|
||||
**metrics,
|
||||
"run_alloc_peak": run_ap,
|
||||
"run_res_peak": run_rp,
|
||||
}
|
||||
)
|
||||
seq, name = c
|
||||
w(
|
||||
{
|
||||
"kind": "ack",
|
||||
"t": round(time.time(), 4),
|
||||
"m": round(time.monotonic(), 4),
|
||||
"window": name,
|
||||
"seq": seq,
|
||||
}
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 - telemetry must never raise into the worker
|
||||
w(
|
||||
{
|
||||
"kind": "error",
|
||||
"t": round(time.time(), 4),
|
||||
"m": round(time.monotonic(), 4),
|
||||
"window": name,
|
||||
"seq": seq,
|
||||
"error": f"{type(e).__name__}: {e}"[:300],
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,488 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Per-rank determinism-toggle line (log-only; changes no behaviour).
|
||||
|
||||
Why this exists: the effective state of the
|
||||
four determinism toggles and the three persistent_topk switches must be
|
||||
attributable to a RANK and a PID. The kernel's own stderr line
|
||||
(``[merlin] persistent_topk tie policy: ...``, topk.cu report_tie_policy_once)
|
||||
is printed once per process with no tag, and FULL_K / QSA_SORT have no log
|
||||
producer at all. Every GPU worker therefore logs exactly one line::
|
||||
|
||||
e2-toggles[PP<p>_TP<t>]: pid=<pid> MERLIN_FULL_K=<raw> MERLIN_QSA_SORT=<raw>
|
||||
MERLIN_TIE_DET=<raw> MERLIN_TIE_RECENT=<raw> MERLIN_TOPK_EXACT_MEDIUM=<raw>
|
||||
MERLIN_TOPK_EXACT_DECODE=<raw> MERLIN_TOPK_DET_LARGE=<raw>
|
||||
MERLIN_TOPK_SORTED_EMIT=<raw>
|
||||
effective: full_k=<0|1> qsa_sort=<0|1|invalid> tie_det=<0|1> tie_recent=<0|1>
|
||||
tie_recent_effective=<0|1> exact_medium=<0|1> exact_decode=<0|1>
|
||||
det_large=<0|1> sorted_emit=<0|1> source=<libc-getenv|os-environ>
|
||||
cpp-cache: <full_k=... tie_det=... tie_recent=... tie_recent_effective=...
|
||||
exact_medium=... exact_decode=... det_large=... sorted_emit=...
|
||||
source=cpp-cache | n-a reason=<why>>
|
||||
python: qsa_sort=<0|1|n-a> sorted_emit_skip=<0|1|n-a>
|
||||
source=qsa_indexer._QSA_SORT
|
||||
device: switch=<0|1|n-a> switch_large=<0|1|n-a> source=counter-slots-8-9
|
||||
|
||||
(one line in the log). ``<raw>`` is ``unset``, a plain token, or a double-quoted
|
||||
string with every byte outside ``[!-~]`` (and ``"`` / ``\\``) hex-escaped, so
|
||||
the line always splits on whitespace.
|
||||
|
||||
WHAT EACH SECTION IS
|
||||
* ``effective:`` -- the environment at emission time, parsed with the SAME
|
||||
rules the readers use (C++ ``std::getenv`` + ``std::atoi != 0`` for the six
|
||||
C++ names, ``bool(int(...))`` for QSA_SORT; file:line of each below). The
|
||||
C++ names are read through libc ``getenv`` -- the ``environ`` that
|
||||
``std::getenv`` reads -- not ``os.environ``, which can diverge from it after
|
||||
an ``os.putenv``. A MIRROR of the parse, not the kernel's state. It exists
|
||||
to report the rule; nothing may branch on it (vllm/envs.py "NOT A PYTHON
|
||||
READ SITE" still holds: this module decides nothing).
|
||||
* ``cpp-cache:`` -- the kernel's switches as the C++ statics cached them, from
|
||||
``torch.ops._C.persistent_topk_toggle_state`` and
|
||||
``torch.ops._moe_C.marlin_moe_full_k_state`` (query ops that only LOAD a
|
||||
mirror the static initialisers publish; they never fill a cache). ``n-a``
|
||||
with a reason on a build without those ops -- the extensions this release
|
||||
was first cut against (earlier v2.2 builds, not published) have none.
|
||||
* ``python:`` -- ``qsa_indexer._QSA_SORT``, the module constant the QSA sort
|
||||
actually branches on (looked up in ``sys.modules``, never imported here;
|
||||
``n-a`` if the module was never loaded in this process), and
|
||||
``sorted_emit_skip`` -- ``qsa_indexer._kernel_sorted_emit_state``, i.e.
|
||||
whether the Python sort is being SKIPPED because the kernel reported
|
||||
MERLIN_TOPK_SORTED_EMIT through ``persistent_topk_toggle_state`` field 6
|
||||
(``n-a`` until the first QSA top-k has run and latched it).
|
||||
* ``device:`` -- counter slots [8]/[9], which the counter op fills from the
|
||||
C++ ``exact_medium_enabled()`` / ``det_large_enabled()`` statics
|
||||
(topk.cu persistent_topk_exact_counters).
|
||||
|
||||
WHEN. Once per worker, after ``compile_or_warm_up_model`` has run the MRV2
|
||||
kernel warmup and cudagraph capture (``arm()``), i.e. after this process's
|
||||
first persistent_topk and Marlin MoE launches -- NOT at the counter reader's
|
||||
first read: an integration boot logged its first counter line BEFORE
|
||||
the first tie-policy line in the engine log, because the
|
||||
first ``maybe_log`` runs at the head of the warmup's first execute step, before
|
||||
that step's forward. When the cpp-cache ops exist and a static is still
|
||||
uninitialised at ``arm()``, the line is deferred to the head of the first
|
||||
execute step where every static is filled, and forced out at shutdown
|
||||
otherwise (with ``uninit`` values, which the smoke FAILs). It does not depend on
|
||||
``MERLIN_TOPK_EXACT_COUNTERS_LOG``: it fires with the counter log off.
|
||||
|
||||
WHAT THIS DOES NOT PROVE. Without the cpp-cache ops the C++ state is inferred,
|
||||
not read: for FULL_K, TIE_DET, TIE_RECENT and EXACT_DECODE the line is the
|
||||
environment at emission time run through the C++ parser's rules, and the only
|
||||
C++-derived evidence is ``device:`` (EXACT_MEDIUM and DET_LARGE) plus the
|
||||
untagged stderr policy line. The residual gap is an environment mutated
|
||||
between the kernel's first ``getenv`` and this read. Nothing in this tree does
|
||||
that (by grep of csrc/): no ``setenv``/``putenv``/``unsetenv`` under csrc/, and
|
||||
the only runtime ``os.environ.update`` calls in vllm/ are the RunAI streamer's
|
||||
fixed ``RUNAI_STREAMER_*`` keys (model_executor/model_loader/
|
||||
runai_streamer_loader.py:73) and the Ray executor's ``override_env_vars``
|
||||
(v1/executor/ray_utils.py:180), which the multiproc executor never calls. With the
|
||||
cpp-cache ops that gap closes for every C++ name. Even then the line proves the
|
||||
CONFIGURATION each rank ran, not that a switch changed any output: that is the
|
||||
job of an output-equality run (identical sends -> one distinct output).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# (environment name, effective key, value when unset or empty) -- the C++
|
||||
# readers, each `static const bool enabled = [] { ... }();` once per process:
|
||||
# MERLIN_FULL_K csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu:57-63
|
||||
# MERLIN_TIE_DET csrc/libtorch_stable/topk.cu:60-66
|
||||
# MERLIN_TIE_RECENT csrc/libtorch_stable/topk.cu:52-58
|
||||
# MERLIN_TOPK_EXACT_MEDIUM csrc/libtorch_stable/topk.cu:68-74 (unset -> ON)
|
||||
# MERLIN_TOPK_EXACT_DECODE csrc/libtorch_stable/topk.cu:103-109
|
||||
# MERLIN_TOPK_DET_LARGE csrc/libtorch_stable/topk.cu:86-92 (unset -> ON)
|
||||
# MERLIN_TOPK_SORTED_EMIT csrc/libtorch_stable/topk.cu sorted_emit_enabled
|
||||
CXX_SWITCHES: tuple[tuple[str, str, bool], ...] = (
|
||||
("MERLIN_FULL_K", "full_k", False),
|
||||
("MERLIN_TIE_DET", "tie_det", False),
|
||||
("MERLIN_TIE_RECENT", "tie_recent", False),
|
||||
("MERLIN_TOPK_EXACT_MEDIUM", "exact_medium", True),
|
||||
("MERLIN_TOPK_EXACT_DECODE", "exact_decode", False),
|
||||
("MERLIN_TOPK_DET_LARGE", "det_large", True),
|
||||
("MERLIN_TOPK_SORTED_EMIT", "sorted_emit", False),
|
||||
)
|
||||
QSA_SORT_NAME = "MERLIN_QSA_SORT"
|
||||
RAW_ORDER = (
|
||||
"MERLIN_FULL_K",
|
||||
"MERLIN_QSA_SORT",
|
||||
"MERLIN_TIE_DET",
|
||||
"MERLIN_TIE_RECENT",
|
||||
"MERLIN_TOPK_EXACT_MEDIUM",
|
||||
"MERLIN_TOPK_EXACT_DECODE",
|
||||
"MERLIN_TOPK_DET_LARGE",
|
||||
"MERLIN_TOPK_SORTED_EMIT",
|
||||
)
|
||||
EFFECTIVE_ORDER = (
|
||||
"full_k",
|
||||
"qsa_sort",
|
||||
"tie_det",
|
||||
"tie_recent",
|
||||
"tie_recent_effective",
|
||||
"exact_medium",
|
||||
"exact_decode",
|
||||
"det_large",
|
||||
"sorted_emit",
|
||||
)
|
||||
CPP_ORDER = tuple(k for k in EFFECTIVE_ORDER if k != "qsa_sort")
|
||||
|
||||
_QSA_MODULE = "vllm.models.qwen4_exp.nvidia.ops.qsa_indexer"
|
||||
_C_SPACE = b" \t\n\v\f\r" # isspace() in the "C" locale
|
||||
_LONG_MIN, _LONG_MAX = -(2**63), 2**63 - 1 # LP64 long, the reference host's glibc
|
||||
|
||||
|
||||
# --------------------------------------------------------------- parsers ----
|
||||
def c_atoi(raw: bytes) -> int:
|
||||
"""glibc ``atoi``: ``(int) strtol(raw, NULL, 10)`` (glibc stdlib/atoi.c).
|
||||
|
||||
Leading C-locale whitespace, one optional sign, then decimal digits up to
|
||||
the first non-digit; no digits -> 0. ``strtol`` clamps to the ``long``
|
||||
range and the ``(int)`` conversion keeps the low 32 bits (GCC's documented
|
||||
modulo behaviour). So ``"true"`` -> 0, ``"1abc"`` -> 1, ``"0x1"`` -> 0,
|
||||
``" +1"`` -> 1, ``"+ 1"`` -> 0 and ``"4294967296"`` -> 0.
|
||||
"""
|
||||
i, n = 0, len(raw)
|
||||
while i < n and raw[i] in _C_SPACE:
|
||||
i += 1
|
||||
negative = False
|
||||
if i < n and raw[i] in b"+-":
|
||||
negative = raw[i] == ord("-")
|
||||
i += 1
|
||||
value, digits = 0, 0
|
||||
while i < n and 0x30 <= raw[i] <= 0x39:
|
||||
value = value * 10 + (raw[i] - 0x30)
|
||||
digits += 1
|
||||
i += 1
|
||||
if digits == 0:
|
||||
return 0
|
||||
if negative:
|
||||
value = -value
|
||||
value = max(_LONG_MIN, min(_LONG_MAX, value))
|
||||
value &= 0xFFFFFFFF
|
||||
return value - (1 << 32) if value >= (1 << 31) else value
|
||||
|
||||
|
||||
def cxx_switch(raw: bytes | None, unset_value: bool) -> bool:
|
||||
"""The C++ getter body, both shapes:
|
||||
default OFF ``(raw != nullptr && raw[0] != '\\0') ? (atoi(raw) != 0) : false``
|
||||
default ON ``(raw == nullptr || raw[0] == '\\0') ? true : (atoi(raw) != 0)``
|
||||
An environment value cannot contain NUL, so "empty" is exactly
|
||||
``raw[0] == '\\0'``."""
|
||||
if raw is None or raw == b"":
|
||||
return unset_value
|
||||
return c_atoi(raw) != 0
|
||||
|
||||
|
||||
def qsa_sort_value(raw: str | None) -> str:
|
||||
"""vllm/envs.py:2372 ``bool(int(os.getenv("MERLIN_QSA_SORT", "0")))``,
|
||||
read once at import by qsa_indexer.py:51 (``_QSA_SORT``). ``int()`` raises
|
||||
on ``"true"`` and on ``""`` -- that engine would not have booted -- which is
|
||||
reported as ``invalid`` rather than guessed. Python's own ``int`` is used,
|
||||
so whitespace, ``+``/``-``, ``1_0`` and non-ASCII digits parse exactly as
|
||||
the reader parses them."""
|
||||
try:
|
||||
return "1" if bool(int("0" if raw is None else raw)) else "0"
|
||||
except ValueError:
|
||||
return "invalid"
|
||||
|
||||
|
||||
def effective_tie_recent(det_ties: bool, tie_recent: bool) -> bool:
|
||||
"""csrc/libtorch_stable/persistent_topk.cuh:65-68: ``det_ties ? tie_recent
|
||||
: true`` -- with TIE_DET off the restart paths keep most-recent-wins."""
|
||||
return tie_recent if det_ties else True
|
||||
|
||||
|
||||
# --------------------------------------------------------------- sources ----
|
||||
def _libc_getenv() -> tuple[Callable[[str], bytes | None], str]:
|
||||
"""libc ``getenv``: the ``environ`` the C++ ``std::getenv`` calls read.
|
||||
|
||||
Falls back to ``os.environb`` / ``os.environ`` -- and says so in the line's
|
||||
``source=`` -- where there is no C library to ask (an offline run off Linux).
|
||||
"""
|
||||
try:
|
||||
import ctypes
|
||||
|
||||
getenv = ctypes.CDLL(None).getenv
|
||||
getenv.restype = ctypes.c_char_p
|
||||
getenv.argtypes = [ctypes.c_char_p]
|
||||
|
||||
def read(name: str) -> bytes | None:
|
||||
return getenv(name.encode("ascii"))
|
||||
|
||||
return read, "libc-getenv"
|
||||
except (OSError, AttributeError, TypeError):
|
||||
pass
|
||||
environb = getattr(os, "environb", None)
|
||||
if environb is not None:
|
||||
return (lambda name: environb.get(name.encode("ascii"))), "os-environ"
|
||||
|
||||
def read_environ(name: str) -> bytes | None:
|
||||
value = os.environ.get(name)
|
||||
return None if value is None else value.encode("utf-8", "surrogateescape")
|
||||
|
||||
return read_environ, "os-environ"
|
||||
|
||||
|
||||
def _printable(b: int) -> bool:
|
||||
return 0x21 <= b <= 0x7E and b not in b'"\\'
|
||||
|
||||
|
||||
def encode_raw(raw: bytes | None) -> str:
|
||||
"""``unset``, a plain token, or a quoted, hex-escaped string -- never a
|
||||
space, so the line splits on whitespace. The literal value ``unset`` is
|
||||
quoted so it cannot pass for an absent variable."""
|
||||
if raw is None:
|
||||
return "unset"
|
||||
if raw and raw != b"unset" and all(_printable(b) for b in raw):
|
||||
return raw.decode("ascii")
|
||||
return '"' + "".join(chr(b) if _printable(b) else f"\\x{b:02x}" for b in raw) + '"'
|
||||
|
||||
|
||||
def environ_view(
|
||||
cxx_getenv: Callable[[str], bytes | None],
|
||||
py_getenv: Callable[[str], str | None],
|
||||
) -> tuple[dict[str, str], dict[str, str]]:
|
||||
"""(raw value by name, effective map) from one read of the environment."""
|
||||
raw: dict[str, str] = {}
|
||||
flags: dict[str, bool] = {}
|
||||
for name, key, unset_value in CXX_SWITCHES:
|
||||
value = cxx_getenv(name)
|
||||
raw[name] = encode_raw(value)
|
||||
flags[key] = cxx_switch(value, unset_value)
|
||||
qsa_raw = py_getenv(QSA_SORT_NAME)
|
||||
raw[QSA_SORT_NAME] = encode_raw(
|
||||
None if qsa_raw is None else qsa_raw.encode("utf-8", "surrogateescape")
|
||||
)
|
||||
eff = {key: "1" if flags[key] else "0" for key in flags}
|
||||
eff["qsa_sort"] = qsa_sort_value(qsa_raw)
|
||||
eff["tie_recent_effective"] = (
|
||||
"1" if effective_tie_recent(flags["tie_det"], flags["tie_recent"]) else "0"
|
||||
)
|
||||
return raw, {key: eff[key] for key in EFFECTIVE_ORDER}
|
||||
|
||||
|
||||
_TOPK_STATE_FIELDS = (
|
||||
"tie_det",
|
||||
"tie_recent",
|
||||
"tie_recent_effective",
|
||||
"exact_medium",
|
||||
"exact_decode",
|
||||
"det_large",
|
||||
"sorted_emit",
|
||||
)
|
||||
# Known layouts: version -> field count. Layout 2 is layout 1
|
||||
# plus field 6 (MERLIN_TOPK_SORTED_EMIT); fields 0-5 are unchanged.
|
||||
_TOPK_STATE_LAYOUTS = {1: 6, 2: 7}
|
||||
|
||||
|
||||
def decode_topk_state(state: int) -> dict[str, str] | str:
|
||||
"""Unpack ``persistent_topk_toggle_state()``: 2 bits per field (0 off,
|
||||
1 on, 2 not initialised), field count at bits 48-55, layout version at
|
||||
bits 56-62 -- (1, 6) or (2, 7). Any other layout is refused with a reason
|
||||
string. A layout-1 build has no sorted emission, reported as ``n-a``."""
|
||||
version, count = (state >> 56) & 0x7F, (state >> 48) & 0xFF
|
||||
if _TOPK_STATE_LAYOUTS.get(version) != count:
|
||||
return f"unknown-layout-v{version}-{count}-fields"
|
||||
names = {0: "0", 1: "1", 2: "uninit"}
|
||||
out = {
|
||||
key: names.get((state >> (2 * i)) & 3, "invalid")
|
||||
for i, key in enumerate(_TOPK_STATE_FIELDS[:count])
|
||||
}
|
||||
for key in _TOPK_STATE_FIELDS[count:]:
|
||||
out[key] = "n-a"
|
||||
return out
|
||||
|
||||
|
||||
def decode_full_k_state(state: int) -> str:
|
||||
"""``marlin_moe_full_k_state()``: -1 not initialised, 0 off, 1 on."""
|
||||
return {-1: "uninit", 0: "0", 1: "1"}.get(int(state), "invalid")
|
||||
|
||||
|
||||
def _op(namespace: str, name: str):
|
||||
try:
|
||||
import torch
|
||||
|
||||
return getattr(getattr(torch.ops, namespace), name, None)
|
||||
except (AttributeError, RuntimeError, ImportError):
|
||||
return None
|
||||
|
||||
|
||||
def cpp_cache() -> dict[str, str] | str:
|
||||
"""The C++ cached switches, or a reason string when the build cannot say."""
|
||||
topk_op = _op("_C", "persistent_topk_toggle_state")
|
||||
full_k_op = _op("_moe_C", "marlin_moe_full_k_state")
|
||||
missing = [
|
||||
name
|
||||
for name, op in (
|
||||
("_C.persistent_topk_toggle_state", topk_op),
|
||||
("_moe_C.marlin_moe_full_k_state", full_k_op),
|
||||
)
|
||||
if op is None
|
||||
]
|
||||
if missing:
|
||||
return "op-absent:" + ",".join(missing)
|
||||
decoded = decode_topk_state(int(topk_op()))
|
||||
if isinstance(decoded, str):
|
||||
return decoded
|
||||
out = dict(decoded, full_k=decode_full_k_state(int(full_k_op())))
|
||||
return {key: out[key] for key in CPP_ORDER}
|
||||
|
||||
|
||||
def qsa_sort_live() -> str:
|
||||
module = sys.modules.get(_QSA_MODULE)
|
||||
if module is None or not hasattr(module, "_QSA_SORT"):
|
||||
return "n-a"
|
||||
return "1" if bool(module._QSA_SORT) else "0"
|
||||
|
||||
|
||||
def sorted_emit_skip_live() -> str:
|
||||
"""``qsa_indexer._kernel_sorted_emit_state``: 1 = the Python sort is being
|
||||
skipped because the kernel reported sorted emission, 0 = not skipped,
|
||||
``n-a`` = not latched yet (or the module was never loaded)."""
|
||||
module = sys.modules.get(_QSA_MODULE)
|
||||
state = getattr(module, "_kernel_sorted_emit_state", None) if module else None
|
||||
if state is None:
|
||||
return "n-a"
|
||||
return "1" if state else "0"
|
||||
|
||||
|
||||
def device_switches() -> dict[str, str]:
|
||||
"""Counter slots [8]/[9] (switch / switch_large). One synchronizing read
|
||||
(topk_counters.read refuses inside a capture), so it is only called from
|
||||
arm() and the head of an execute step."""
|
||||
try:
|
||||
from vllm.models.qwen4_exp.nvidia.ops import topk_counters
|
||||
|
||||
counters = topk_counters.read()
|
||||
except Exception:
|
||||
counters = {}
|
||||
return {
|
||||
key: str(counters[key]) if key in counters else "n-a"
|
||||
for key in ("switch", "switch_large")
|
||||
}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ line ----
|
||||
def toggles_line(
|
||||
tag: str,
|
||||
pid: int,
|
||||
raw: dict[str, str],
|
||||
effective: dict[str, str],
|
||||
env_source: str,
|
||||
cpp: dict[str, str] | str,
|
||||
qsa_live: str,
|
||||
device: dict[str, str],
|
||||
sorted_skip: str = "n-a",
|
||||
) -> str:
|
||||
parts = [f"e2-toggles[{tag}]: pid={pid}"]
|
||||
parts += [f"{name}={raw[name]}" for name in RAW_ORDER]
|
||||
parts.append("effective:")
|
||||
parts += [f"{key}={effective[key]}" for key in EFFECTIVE_ORDER]
|
||||
parts.append(f"source={env_source}")
|
||||
parts.append("cpp-cache:")
|
||||
if isinstance(cpp, str):
|
||||
parts += ["n-a", f"reason={cpp}"]
|
||||
else:
|
||||
parts += [f"{key}={cpp[key]}" for key in CPP_ORDER]
|
||||
parts.append("source=cpp-cache")
|
||||
parts += [
|
||||
"python:",
|
||||
f"qsa_sort={qsa_live}",
|
||||
f"sorted_emit_skip={sorted_skip}",
|
||||
"source=qsa_indexer._QSA_SORT",
|
||||
]
|
||||
parts.append("device:")
|
||||
parts += [f"{key}={device[key]}" for key in ("switch", "switch_large")]
|
||||
parts.append("source=counter-slots-8-9")
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def _rank_tag() -> str:
|
||||
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
|
||||
|
||||
return f"PP{get_pp_group().rank_in_group}_TP{get_tp_group().rank_in_group}"
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- state ----
|
||||
_armed = False
|
||||
_emitted = False
|
||||
_deferral_noted = False
|
||||
|
||||
|
||||
def _reset_for_tests() -> None:
|
||||
global _armed, _emitted, _deferral_noted
|
||||
_armed = _emitted = _deferral_noted = False
|
||||
|
||||
|
||||
def arm(**kwargs) -> str:
|
||||
"""Once per worker, at the end of compile_or_warm_up_model (warmup and
|
||||
capture done): emit now, unless a C++ static is still uninitialised."""
|
||||
global _armed
|
||||
_armed = True
|
||||
return maybe_emit(**kwargs)
|
||||
|
||||
|
||||
def maybe_emit(
|
||||
force: bool = False,
|
||||
*,
|
||||
tag_fn: Callable[[], str] = _rank_tag,
|
||||
getenv_fn: Callable[[], tuple[Callable[[str], bytes | None], str]] = _libc_getenv,
|
||||
cpp_fn: Callable[[], dict[str, str] | str] = cpp_cache,
|
||||
device_fn: Callable[[], dict[str, str]] = device_switches,
|
||||
qsa_fn: Callable[[], str] = qsa_sort_live,
|
||||
sorted_skip_fn: Callable[[], str] = sorted_emit_skip_live,
|
||||
) -> str:
|
||||
"""Emit the line at most once per process. One bool check when already
|
||||
emitted or not yet armed, so it can sit at the head of every execute step.
|
||||
Never raises. Returns the line when this call emitted it, else ""."""
|
||||
global _emitted, _deferral_noted
|
||||
if _emitted or not _armed:
|
||||
return ""
|
||||
tag = "PP?_TP?"
|
||||
try:
|
||||
tag = tag_fn()
|
||||
cpp = cpp_fn()
|
||||
pending = (
|
||||
sorted(k for k, v in cpp.items() if v == "uninit")
|
||||
if isinstance(cpp, dict)
|
||||
else []
|
||||
)
|
||||
if pending and not force:
|
||||
if not _deferral_noted:
|
||||
_deferral_noted = True
|
||||
# Deliberately not 'e2-toggles[': the smoke counts those.
|
||||
logger.info(
|
||||
"e2-toggles deferred on %s: C++ cache not initialised for %s",
|
||||
tag,
|
||||
",".join(pending),
|
||||
)
|
||||
return ""
|
||||
cxx_getenv, env_source = getenv_fn()
|
||||
raw, effective = environ_view(cxx_getenv, os.environ.get)
|
||||
line = toggles_line(
|
||||
tag,
|
||||
os.getpid(),
|
||||
raw,
|
||||
effective,
|
||||
env_source,
|
||||
cpp,
|
||||
qsa_fn(),
|
||||
device_fn(),
|
||||
sorted_skip_fn(),
|
||||
)
|
||||
_emitted = True
|
||||
logger.info("%s", line)
|
||||
return line
|
||||
except Exception as error: # log-only: must never break a boot or a step
|
||||
_emitted = True
|
||||
logger.warning(
|
||||
"e2-toggles[%s]: unavailable (%s: %s)", tag, type(error).__name__, error
|
||||
)
|
||||
return ""
|
||||
@@ -6,6 +6,7 @@ import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker import guard_telemetry
|
||||
from vllm.v1.worker.gpu.buffer_utils import (
|
||||
FusedStagedWriter,
|
||||
StagedWriteTensor,
|
||||
@@ -13,6 +14,12 @@ from vllm.v1.worker.gpu.buffer_utils import (
|
||||
_load_ptr,
|
||||
)
|
||||
|
||||
# Bias so a negative row/index survives atomic_max on a zeroed counter
|
||||
# -- it MUST be a tl.constexpr: a plain module-level int is not readable
|
||||
# from a @triton.jit body -- an earlier build shipped one and every guarded kernel died
|
||||
# with "NameError: Cannot access global variable _GUARD_BIAS" on the smoke.
|
||||
_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS)
|
||||
|
||||
|
||||
class BlockTables:
|
||||
# Set by the elastic EP warmup so KV writes land in the null block.
|
||||
@@ -180,6 +187,9 @@ class BlockTables:
|
||||
self.num_blocks.gpu,
|
||||
self.num_blocks.gpu.stride(0),
|
||||
num_reqs,
|
||||
guard_telemetry.counter(
|
||||
guard_telemetry.SITE_BT_GATHER, idx_mapping.device
|
||||
),
|
||||
BLOCK_SIZE=1024, # type: ignore
|
||||
)
|
||||
return tuple(bt[:num_reqs_padded] for bt in out)
|
||||
@@ -217,6 +227,10 @@ class BlockTables:
|
||||
positions,
|
||||
self.block_table_ptrs,
|
||||
self.block_table_strides,
|
||||
self.num_blocks.gpu.stride(0),
|
||||
guard_telemetry.counter(
|
||||
guard_telemetry.SITE_SLOT_MAPPING, idx_mapping.device
|
||||
),
|
||||
self.block_sizes_tensor,
|
||||
self.kernel_block_sizes_tensor,
|
||||
self.slot_mapping_enabled,
|
||||
@@ -252,6 +266,7 @@ def _gather_block_tables_kernel(
|
||||
num_blocks_ptr, # [num_kv_cache_groups, max_num_reqs]
|
||||
num_blocks_stride,
|
||||
num_reqs, # actual number of requests (for padding)
|
||||
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
# kv cache group id
|
||||
@@ -271,8 +286,39 @@ def _gather_block_tables_kernel(
|
||||
return
|
||||
|
||||
req_idx = tl.load(batch_idx_to_req_idx + batch_idx)
|
||||
# Bounds-guard instrumentation (guard 3): req_idx indexes both num_blocks_ptr and the
|
||||
# persistent source table, which have max_num_reqs rows. num_blocks_stride
|
||||
# IS max_num_reqs, so the bound needs no new argument. Zero the destination
|
||||
# row and return, mirroring the padded-row path above.
|
||||
if (req_idx < 0) or (req_idx >= num_blocks_stride):
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (batch_idx).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, (req_idx).to(tl.int64) + _GUARD_BIAS)
|
||||
for i in tl.range(0, max_num_blocks, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
tl.store(dst_row_ptr + offset, 0, mask=offset < max_num_blocks)
|
||||
return
|
||||
|
||||
group_num_blocks_ptr = num_blocks_ptr + group_id * num_blocks_stride
|
||||
num_blocks = tl.load(group_num_blocks_ptr + req_idx)
|
||||
# A row count wider than the table would copy past the end of both rows.
|
||||
#
|
||||
# Clamp UNCONDITIONALLY and record conditionally. Doing the clamp inside
|
||||
# the `if` rebound an int32 name (num_blocks, loaded from num_blocks_ptr)
|
||||
# to int64 (max_num_blocks is `stride`, loaded from block_table_strides),
|
||||
# and Triton requires a name to keep its type across a branch:
|
||||
# "initial value for `num_blocks` is of type int32[], but the then block
|
||||
# redefines it as int64[]"
|
||||
# That killed an early boot in warmup, at the first real gather. The
|
||||
# explicit .to(tl.int32) pins the type rather than relying on whatever the
|
||||
# operands promote to.
|
||||
if (num_blocks < 0) or (num_blocks > max_num_blocks):
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (batch_idx).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, (num_blocks).to(tl.int64) + _GUARD_BIAS)
|
||||
num_blocks = tl.minimum(
|
||||
tl.maximum(num_blocks, 0), max_num_blocks
|
||||
).to(tl.int32)
|
||||
|
||||
src_block_table_ptr = _load_ptr(src_block_table_ptrs + group_id, tl.int32)
|
||||
src_row_ptr = src_block_table_ptr + req_idx * stride
|
||||
@@ -291,6 +337,8 @@ def _compute_slot_mappings_kernel(
|
||||
pos, # [num_tokens]
|
||||
block_table_ptrs, # [num_kv_cache_groups]
|
||||
block_table_strides, # [num_kv_cache_groups]
|
||||
bt_num_rows, # bounds-guard instrumentation: rows in the persistent block tables
|
||||
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||
block_sizes, # [num_kv_cache_groups]
|
||||
kernel_block_sizes, # [num_kv_cache_groups]
|
||||
slot_mapping_enabled, # [num_kv_cache_groups]
|
||||
@@ -328,9 +376,27 @@ def _compute_slot_mappings_kernel(
|
||||
sharded = tl.load(dcp_sharded + group_id)
|
||||
|
||||
req_state_idx = tl.load(idx_mapping + batch_idx)
|
||||
# Bounds-guard instrumentation (guard 4): req_state_idx indexes the persistent block table
|
||||
# before any bound is applied. An invalid row writes PAD_ID for this
|
||||
# request rather than reading an arbitrary row.
|
||||
# idx_mapping == -1 marks a dummy (or CUDA-graph padding) request that owns
|
||||
# no blocks: never read its block-table row and emit PAD for its tokens.
|
||||
is_real_req = req_state_idx >= 0
|
||||
|
||||
# Bounds-guard instrumentation (guard 4): req_state_idx indexes the persistent block table
|
||||
# before any bound is applied. An invalid row writes PAD_ID for this
|
||||
# request rather than reading an arbitrary row.
|
||||
if (req_state_idx < 0) or (req_state_idx >= bt_num_rows):
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (batch_idx).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, (req_state_idx).to(tl.int64) + _GUARD_BIAS)
|
||||
bad_start = tl.load(query_start_loc + batch_idx)
|
||||
bad_end = tl.load(query_start_loc + batch_idx + 1)
|
||||
for i in range(bad_start, bad_end, TRITON_BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, TRITON_BLOCK_SIZE)
|
||||
tl.store(slot_mapping_ptr + offset, PAD_ID, mask=offset < bad_end)
|
||||
return
|
||||
|
||||
start_idx = tl.load(query_start_loc + batch_idx)
|
||||
end_idx = tl.load(query_start_loc + batch_idx + 1)
|
||||
for i in range(start_idx, end_idx, TRITON_BLOCK_SIZE):
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import contextlib
|
||||
import gc
|
||||
import itertools
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass, replace
|
||||
from itertools import groupby, product
|
||||
@@ -13,6 +14,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
from tqdm import tqdm
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.breakable_cudagraph import (
|
||||
BreakableCUDAGraphWrapper,
|
||||
is_breakable_cudagraph_enabled,
|
||||
@@ -33,6 +35,9 @@ from vllm.distributed.parallel_state import (
|
||||
)
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.ple_offload_layer import (
|
||||
observe_ple_copy_geometry,
|
||||
)
|
||||
from vllm.model_executor.offloader.base import get_offloader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -173,6 +178,14 @@ class CudaGraphManager:
|
||||
self._lora_dispatch_map, self._max_lora_case = self._build_lora_dispatch_map()
|
||||
|
||||
self.graphs: dict[BatchExecutionDescriptor, torch.cuda.CUDAGraph] = {}
|
||||
# FULL graphs bake every PLE wait/copy into their native graph. Metadata
|
||||
# comes from the actual custom-op implementation during native capture.
|
||||
self._ple_full_copy_geometry: dict[
|
||||
BatchExecutionDescriptor, tuple[tuple[str, int, int], ...]
|
||||
] = {}
|
||||
self._expected_ple_copy_layers: tuple[str, ...] = ()
|
||||
# Only the target-model manager is allowed to capture PLE operations.
|
||||
self._allow_ple_full_graph = False
|
||||
self.pool = current_platform.get_global_graph_pool() if cudagraph_mode else None
|
||||
|
||||
self._graphs_captured = False
|
||||
@@ -488,6 +501,7 @@ class CudaGraphManager:
|
||||
torch.accelerator.synchronize()
|
||||
free_before = torch.accelerator.get_memory_info()[0]
|
||||
with (
|
||||
self.capture_ple_copy_geometry(desc),
|
||||
capture_pool(self.pool, self.vllm_config) as pool,
|
||||
torch.cuda.graph(
|
||||
graph, pool, stream=self._capture_stream(desc)
|
||||
@@ -514,6 +528,78 @@ class CudaGraphManager:
|
||||
{desc.num_tokens for desc in self.graphs if desc.num_active_loras == 0}
|
||||
)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def capture_ple_copy_geometry(
|
||||
self, desc: BatchExecutionDescriptor
|
||||
) -> Iterator[None]:
|
||||
"""Production capture callback for actual PLE custom-op geometry."""
|
||||
with observe_ple_copy_geometry() as geometry:
|
||||
yield
|
||||
|
||||
if geometry and not self._allow_ple_full_graph:
|
||||
raise RuntimeError(
|
||||
"PLE wait-copy reached an unsupported FULL graph path; "
|
||||
"speculator/draft FULL graphs may not contain PLE operations"
|
||||
)
|
||||
if geometry:
|
||||
self.record_ple_copy_geometry(desc, geometry)
|
||||
elif self._expected_ple_copy_layers:
|
||||
raise RuntimeError(
|
||||
f"PLE copy geometry was not observed while capturing FULL graph {desc}"
|
||||
)
|
||||
|
||||
def record_ple_copy_geometry(
|
||||
self,
|
||||
desc: BatchExecutionDescriptor,
|
||||
geometry: Iterable[tuple[str, int, int]],
|
||||
) -> None:
|
||||
if desc.cg_mode != CUDAGraphMode.FULL:
|
||||
raise ValueError("PLE capture geometry is recorded only for FULL graphs")
|
||||
observed = tuple(
|
||||
(str(layer), int(rows), int(nbytes))
|
||||
for layer, rows, nbytes in geometry
|
||||
)
|
||||
if not observed:
|
||||
raise RuntimeError(f"PLE FULL graph {desc} observed no wait-copy operations")
|
||||
observed_layers = tuple(layer for layer, _, _ in observed)
|
||||
expected_layers = self._expected_ple_copy_layers
|
||||
if expected_layers and sorted(observed_layers) != sorted(expected_layers):
|
||||
raise RuntimeError(
|
||||
f"PLE FULL graph {desc} observed layers {observed_layers}, "
|
||||
f"expected {expected_layers}"
|
||||
)
|
||||
inconsistent = [
|
||||
(layer, rows, nbytes)
|
||||
for layer, rows, nbytes in observed
|
||||
if rows != desc.num_tokens or nbytes < 0
|
||||
]
|
||||
if inconsistent:
|
||||
raise RuntimeError(
|
||||
f"PLE FULL graph {desc} actual PLE copy geometry "
|
||||
f"{inconsistent} does not match descriptor rows={desc.num_tokens}"
|
||||
)
|
||||
previous = self._ple_full_copy_geometry.get(desc)
|
||||
if previous is not None and previous != observed:
|
||||
raise RuntimeError(
|
||||
f"PLE FULL graph {desc} changed copy geometry from "
|
||||
f"{previous} to {observed}"
|
||||
)
|
||||
self._ple_full_copy_geometry[desc] = observed
|
||||
|
||||
def ple_copy_rows(self, desc: BatchExecutionDescriptor) -> int:
|
||||
try:
|
||||
geometry = self._ple_full_copy_geometry[desc]
|
||||
except KeyError as error:
|
||||
raise RuntimeError(
|
||||
f"PLE copy geometry was not recorded for FULL graph {desc}"
|
||||
) from error
|
||||
rows = {num_rows for _, num_rows, _ in geometry}
|
||||
if len(rows) != 1:
|
||||
raise RuntimeError(
|
||||
f"PLE FULL graph {desc} has inconsistent per-layer rows: {geometry}"
|
||||
)
|
||||
return rows.pop()
|
||||
|
||||
def dispatch(
|
||||
self,
|
||||
num_reqs: int,
|
||||
@@ -552,6 +638,11 @@ class CudaGraphManager:
|
||||
f"Expected FULL mode, got {desc.cg_mode}"
|
||||
)
|
||||
assert desc in self.graphs, f"No cudagraph for {desc}"
|
||||
if (
|
||||
desc in self._ple_full_copy_geometry
|
||||
and not self._allow_ple_full_graph
|
||||
):
|
||||
raise RuntimeError("refusing unsupported PLE speculator FULL replay")
|
||||
# Sync offloader before replay - needed when transitioning from
|
||||
# eager/piecewise to full cudagraph (e.g., prefill → decode).
|
||||
# The previous eager iteration's start_prefetch may have queued
|
||||
@@ -597,11 +688,32 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
varlen_decode=varlen_decode,
|
||||
ubatch_runner=ubatch_runner,
|
||||
)
|
||||
self._allow_ple_full_graph = True
|
||||
self.hidden_states: torch.Tensor | None = None
|
||||
self.aux_hidden_states: list[torch.Tensor] = []
|
||||
self.use_aux_hidden_state_outputs = False
|
||||
self.intermediate_tensors: IntermediateTensors | None = None
|
||||
|
||||
def _prepare_ple_capture(
|
||||
self, model: nn.Module, *, initialize_breakable: bool
|
||||
) -> None:
|
||||
expected_ple_copy_layers = tuple(
|
||||
str(getattr(module, "_ple_offload_name", name))
|
||||
for name, module in model.named_modules()
|
||||
if getattr(module, "_is_cpu_offloaded", False)
|
||||
and getattr(module, "_host_output_ptr", None) is not None
|
||||
)
|
||||
if (
|
||||
"<unbound>" in expected_ple_copy_layers
|
||||
or len(set(expected_ple_copy_layers)) != len(expected_ple_copy_layers)
|
||||
):
|
||||
raise RuntimeError(
|
||||
"PLE FULL capture requires distinct, bound connector layer identities"
|
||||
)
|
||||
self._expected_ple_copy_layers = expected_ple_copy_layers
|
||||
if initialize_breakable:
|
||||
self.init_breakable_cg_runner(model)
|
||||
|
||||
def capture(
|
||||
self,
|
||||
model: nn.Module,
|
||||
@@ -619,8 +731,9 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
) -> None:
|
||||
"""Capture CUDA graphs for model forward pass."""
|
||||
self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs
|
||||
if self.use_breakable_cg:
|
||||
self.init_breakable_cg_runner(model)
|
||||
self._prepare_ple_capture(
|
||||
model, initialize_breakable=self.use_breakable_cg
|
||||
)
|
||||
|
||||
if self.cudagraph_mode.has_piecewise_cudagraphs() and not (
|
||||
self.use_breakable_cg or has_compiled_submodule(model)
|
||||
@@ -926,7 +1039,9 @@ def profile_cudagraph_memory(runner: "GPUModelRunner") -> int:
|
||||
if manager.use_breakable_cg:
|
||||
# Create the breakable runner before the wrapper pool swap so
|
||||
# its pool is covered as well.
|
||||
manager.init_breakable_cg_runner(runner.model)
|
||||
manager._prepare_ple_capture(
|
||||
runner.model, initialize_breakable=True
|
||||
)
|
||||
all_wrappers = list(CUDAGraphWrapper._all_instances) + list(
|
||||
BreakableCUDAGraphWrapper._all_instances
|
||||
)
|
||||
@@ -1013,12 +1128,46 @@ def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None:
|
||||
runner.cache_config.num_gpu_blocks = minimal_config.num_blocks
|
||||
|
||||
|
||||
def _profiling_kv_cache_bytes(kv_caches: Iterable[Any]) -> int:
|
||||
"""Bytes of the distinct CUDA storages behind the runner's KV cache views."""
|
||||
storages: dict[int, int] = {}
|
||||
for cache in kv_caches:
|
||||
for t in cache if isinstance(cache, (list, tuple)) else (cache,):
|
||||
if isinstance(t, torch.Tensor) and t.is_cuda:
|
||||
st = t.untyped_storage()
|
||||
storages[st.data_ptr()] = st.nbytes()
|
||||
return sum(storages.values())
|
||||
|
||||
|
||||
def _profiling_kv_still_held(
|
||||
kv_bytes: int, allocated_before: int, allocated_after: int
|
||||
) -> int:
|
||||
"""Lower bound on the profiling KV cache bytes still allocated after the
|
||||
teardown: whatever the teardown freed counts against the cache first, so a
|
||||
positive result means something still references the cache (the RecoverSSM
|
||||
commit-context class of leak, ~26 GiB on H200; against a sub-GiB per-card
|
||||
per-card cushion even a small one matters). One-sided: never a false alarm,
|
||||
may miss a leak masked by other frees."""
|
||||
return max(kv_bytes - max(allocated_before - allocated_after, 0), 0)
|
||||
|
||||
|
||||
def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
|
||||
"""Release the profiling KV cache and captured graphs while keeping model
|
||||
weights, so the real ``initialize_kv_cache`` starts from a clean slate."""
|
||||
torch.accelerator.synchronize()
|
||||
kv_bytes = _profiling_kv_cache_bytes(getattr(runner, "kv_caches", ()))
|
||||
allocated_before = torch.cuda.memory_allocated() if kv_bytes else 0
|
||||
if hasattr(runner.model_state, "_mamba_ctx"):
|
||||
runner.model_state._mamba_ctx = None
|
||||
# A PIECEWISE profiling capture records its RecoverSSM step (only FULL
|
||||
# captures skip it), and that step's commit contexts reference the
|
||||
# profiling KV cache.
|
||||
if (recoverssm := getattr(runner.model_state, "recoverssm", None)) is not None:
|
||||
recoverssm.reset()
|
||||
# Under PP the non-last ranks keep detached RecoverSSM steps on the pending
|
||||
# PP entries; they reference the profiling KV cache just the same.
|
||||
if (pp_handler := getattr(runner, "pp_handler", None)) is not None:
|
||||
pp_handler.drop_deferred_states()
|
||||
# Invalidate the align-mode Mamba group metadata cached from the
|
||||
# profiling KVCacheConfig: the real (e.g. PP-projected) config may
|
||||
# place Mamba layers into a different group layout, so it must be
|
||||
@@ -1050,3 +1199,22 @@ def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
|
||||
runner.maybe_remove_all_loras(runner.lora_config)
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
if kv_bytes:
|
||||
# last, after every release above (one-sided check)
|
||||
gc.collect()
|
||||
torch.accelerator.synchronize()
|
||||
held = _profiling_kv_still_held(
|
||||
kv_bytes, allocated_before, torch.cuda.memory_allocated()
|
||||
)
|
||||
if held > (1 << 20):
|
||||
message = (
|
||||
"Profiling teardown: at least %.1f MiB of the %.1f MiB profiling KV "
|
||||
"cache is still allocated; something still references it (it will "
|
||||
"be missing from the real KV cache / free-memory cushion)"
|
||||
) % (held / (1 << 20), kv_bytes / (1 << 20))
|
||||
# Fail boot by default: this memory is absent from the real KV
|
||||
# pool and can invalidate capacity planning. Operators may opt out
|
||||
# temporarily while diagnosing with VLLM_FAIL_ON_PROFILING_KV_LEAK=0.
|
||||
if envs.VLLM_FAIL_ON_PROFILING_KV_LEAK:
|
||||
raise RuntimeError(message)
|
||||
logger.warning(message)
|
||||
|
||||
@@ -53,6 +53,7 @@ from vllm.model_executor.offloader import (
|
||||
set_offloader,
|
||||
)
|
||||
from vllm.model_executor.warmup.jit_warmup import JitWarmupRegistry
|
||||
from vllm.models.qwen4_exp.nvidia.ops import topk_counters
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.encoder_budget import (
|
||||
MultiModalBudget,
|
||||
@@ -64,6 +65,7 @@ from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, async_tensor_h2d
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.e2_guards import assert_resolved_cudagraph_modes
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
MambaSpec,
|
||||
@@ -74,12 +76,16 @@ from vllm.v1.outputs import (
|
||||
ECConnectorOutput,
|
||||
ModelRunnerOutput,
|
||||
)
|
||||
from vllm.v1.ple_offload.connector import PleOffloadConnector
|
||||
from vllm.v1.ple_offload.fatal import fail_ple_execute_model
|
||||
from vllm.v1.ple_offload.test_fault import _FAULT
|
||||
from vllm.v1.watermarking import create_watermarker
|
||||
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
|
||||
from vllm.v1.watermarking.spec_decode import (
|
||||
create_speculative_target_watermarker,
|
||||
speculative_target_watermark_key,
|
||||
)
|
||||
from vllm.v1.worker import e2_toggles, guard_telemetry
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
|
||||
from vllm.v1.worker.gpu import pcp_manager as pcp
|
||||
@@ -174,6 +180,7 @@ from vllm.v1.worker.gpu.ubatch_utils import (
|
||||
maybe_build_ubatch_runner,
|
||||
)
|
||||
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
|
||||
from vllm.v1.worker.ring_tags import clear_ring_row_tags, ring_tag_layers
|
||||
from vllm.v1.worker.utils import (
|
||||
KVBlockZeroer,
|
||||
clear_layer_kv_caches,
|
||||
@@ -185,6 +192,39 @@ from vllm.v1.worker.workspace import lock_workspace, use_workspace_lane
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def grammar_invalid_draft_positions(
|
||||
input_batch: InputBatch,
|
||||
grammar_req_ids: list[str],
|
||||
num_acceptable_drafts: list[int] | None,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor | None:
|
||||
"""Indices of `draft_sampled` entries whose bitmask row is permissive.
|
||||
|
||||
Drafts from `num_acceptable_drafts` on were verified against rows carrying
|
||||
`_full_mask`, so accepting one would sample with no grammar constraint.
|
||||
Returns None when there is nothing to invalidate.
|
||||
"""
|
||||
if not grammar_req_ids or input_batch.num_draft_tokens == 0:
|
||||
return None
|
||||
cu_num_logits = input_batch.cu_num_logits_np.tolist()
|
||||
req_id_to_idx = {req_id: i for i, req_id in enumerate(input_batch.req_ids)}
|
||||
positions: list[int] = []
|
||||
for i, req_id in enumerate(grammar_req_ids):
|
||||
req_idx = req_id_to_idx.get(req_id)
|
||||
if req_idx is None:
|
||||
continue
|
||||
# Without the field (an older scheduler, or warmup) invalidate the whole
|
||||
# window, which is the conservative choice.
|
||||
num_acceptable = (
|
||||
num_acceptable_drafts[i] if num_acceptable_drafts is not None else 0
|
||||
)
|
||||
start = cu_num_logits[req_idx] + 1 + num_acceptable
|
||||
positions.extend(range(start, cu_num_logits[req_idx + 1]))
|
||||
if not positions:
|
||||
return None
|
||||
return torch.tensor(positions, dtype=torch.int64, device=device)
|
||||
|
||||
|
||||
class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
self.vllm_config = vllm_config
|
||||
@@ -203,6 +243,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.jit_warmup_registry = JitWarmupRegistry(vllm_config)
|
||||
|
||||
self.device = device
|
||||
# Bounds-guard instrumentation: allocate the guard counters now, while we are certainly
|
||||
# not capturing. The thread that READS them starts much later, from
|
||||
# gpu_worker after capture. See guard_telemetry.start_reader.
|
||||
guard_telemetry.prealloc(device)
|
||||
self.dtype = self.model_config.dtype
|
||||
self.kv_cache_dtype = self.dtype
|
||||
if self.cache_config.cache_dtype != "auto":
|
||||
@@ -283,7 +327,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.use_aux_hidden_state_outputs = True
|
||||
|
||||
# Draft tokens propagation - for spec-dec + struct outputs.
|
||||
self.draft_tokens_handler = DraftTokensHandler(self.device)
|
||||
self.draft_tokens_handler = DraftTokensHandler(
|
||||
self.device,
|
||||
max_snapshots=max(8, vllm_config.max_concurrent_batches + 1),
|
||||
)
|
||||
|
||||
self.pcp_manager: pcp.PCPManager | None = None
|
||||
|
||||
@@ -341,6 +388,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
# For transferring state from execute_model to subsequent sample_tokens call.
|
||||
self.execute_model_state: ExecuteModelState | None = None
|
||||
# PLE pull transport: set by _setup_ple_offload() from Worker.load_model
|
||||
# when VLLM_E2_PLE_PULL_TRANSPORT=1 and this rank owns a PLE layer.
|
||||
# None otherwise, and every hook below is then a no-op (upstream path).
|
||||
self._ple_offload_connector: PleOffloadConnector | None = None
|
||||
|
||||
# Expert parallelism load balancer.
|
||||
self.eplb = EPLBController(self.parallel_config, self.device)
|
||||
@@ -364,6 +415,26 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
tasks.extend(PoolingRunner.get_supported_tasks(self.model))
|
||||
return tuple(tasks)
|
||||
|
||||
def _setup_ple_offload(self, ipc_addr: str) -> None:
|
||||
"""Initialize PLE offload after the model state is available."""
|
||||
# PLE placeholders are created by model loading, so CUDA output
|
||||
# registration cannot happen in the runner constructor.
|
||||
query_start_loc_source = getattr(self.model_state, "ple_query_start_loc", None)
|
||||
ngram_context_source = getattr(self.model_state, "ngram_context", None)
|
||||
if not isinstance(query_start_loc_source, torch.Tensor):
|
||||
raise RuntimeError("PLE offload requires a query_start_loc source")
|
||||
if not isinstance(ngram_context_source, torch.Tensor):
|
||||
raise RuntimeError("PLE offload requires an ngram_context source")
|
||||
self._ple_offload_connector = PleOffloadConnector(
|
||||
self.vllm_config,
|
||||
self.model,
|
||||
self.device,
|
||||
ipc_addr,
|
||||
input_ids_source=self.input_buffers.input_ids,
|
||||
query_start_loc_source=query_start_loc_source,
|
||||
ngram_context_source=ngram_context_source,
|
||||
)
|
||||
|
||||
def load_model(self, load_dummy_weights: bool = False, *args, **kwargs) -> None:
|
||||
time_before_load = time.perf_counter()
|
||||
if load_dummy_weights:
|
||||
@@ -689,6 +760,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
cp_rank=self.dcp_rank,
|
||||
cp_interleave=self.cp_interleave,
|
||||
)
|
||||
# Mamba "align" indexes a block table from the PP-deferred post-process,
|
||||
# which runs pp_size steps after the gather that produced its
|
||||
# idx_mapping; give the model state the persistent per-req-slot tables.
|
||||
self.model_state.set_block_tables(self.block_tables)
|
||||
self.pcp_manager = pcp.maybe_build_pcp_manager(
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
@@ -729,6 +804,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
is_profiling=is_profiling,
|
||||
piecewise_capture_available=piecewise_capture_available,
|
||||
use_replayssm=self.cache_config.use_replayssm,
|
||||
)
|
||||
self.cudagraph_manager = ModelCudaGraphManager(
|
||||
self.vllm_config,
|
||||
@@ -791,6 +867,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
kernel_block_sizes=self.kernel_block_sizes,
|
||||
static_forward_context=self.compilation_config.static_forward_context,
|
||||
num_blocks=self.kv_cache_config.num_blocks,
|
||||
skip_kv_cache_group_ids=self.kv_cache_config.slot_packed_group_ids,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
@@ -1034,6 +1111,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
@torch.inference_mode()
|
||||
def capture_model(self, *, profile_only: bool = False) -> int:
|
||||
assert self.cudagraph_manager is not None
|
||||
# Fork (0.30 rebase): the construction guard saw the configured
|
||||
# mode; this asserts what the resolver actually handed the target and
|
||||
# drafter managers, before any capture and before the early return.
|
||||
assert_resolved_cudagraph_modes(
|
||||
self.vllm_config, self.cudagraph_manager, self.speculator
|
||||
)
|
||||
capture_encoder = (
|
||||
self.model_state.supports_mm_inputs
|
||||
and self.model_state.encoder_runner.has_cudagraph()
|
||||
@@ -1054,6 +1137,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
torch.accelerator.empty_cache()
|
||||
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
# Decoder capture calls the model outside execute_model. Keep dummy
|
||||
# PLE outputs signaled until all graphs capture their semaphore wait.
|
||||
if self._ple_offload_connector is not None and capture_decoder:
|
||||
self._ple_offload_connector.signal_dummy_outputs(self.max_num_tokens)
|
||||
with self.maybe_setup_dummy_loras(self.lora_config):
|
||||
if capture_encoder:
|
||||
self.model_state.encoder_runner.capture()
|
||||
@@ -1089,6 +1176,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.adaptive_verification.set_initial_cost_curves(timings)
|
||||
self.kv_connector.reset_capture_state()
|
||||
|
||||
if self._ple_offload_connector is not None and capture_decoder:
|
||||
self._ple_offload_connector.release_outputs()
|
||||
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
if not profile_only:
|
||||
@@ -1152,6 +1241,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.req_states.draft_tokens
|
||||
)
|
||||
if outputs is not None:
|
||||
# Restore the step's deferred model state (RecoverSSM commits)
|
||||
# so this postprocess commits the step that sampled them.
|
||||
self.model_state.restore_deferred_state(
|
||||
outputs.pop("deferred_state", None)
|
||||
)
|
||||
self.postprocess_sampled(**outputs)
|
||||
|
||||
def add_requests(self, scheduler_output: SchedulerOutput) -> None:
|
||||
@@ -1240,6 +1334,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
assert self.kv_block_zeroer is not None
|
||||
self.kv_block_zeroer.zero_block_ids(scheduler_output.new_block_ids_to_zero)
|
||||
|
||||
# v2.2 ring-row tag fix: clear the tag words of newly claimed ring blocks. Not
|
||||
# under the zeroing gate; after the zeroer, before copies and forward.
|
||||
if scheduler_output.new_ring_block_ids_to_clear:
|
||||
clear_ring_row_tags(
|
||||
ring_tag_layers(self.compilation_config.static_forward_context),
|
||||
scheduler_output.new_ring_block_ids_to_clear,
|
||||
self.device,
|
||||
)
|
||||
|
||||
# Apply copy-on-write block copies for partial prefix-cache hits, after
|
||||
# zeroing new blocks and before the forward pass reads them.
|
||||
if scheduler_output.kv_cache_block_copies:
|
||||
@@ -1461,6 +1564,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
total_num_logits,
|
||||
self.model_state.num_new_sampled_tokens_per_step,
|
||||
)
|
||||
if self.num_speculative_steps > 0:
|
||||
# Snapshot the drafts that were just written into input_ids, tagged
|
||||
# with this scheduler step. Proposal-time drafts (set later in
|
||||
# sample_tokens) are for the next step and must not overwrite this.
|
||||
self.draft_tokens_handler.snapshot_consumed_drafts(
|
||||
scheduler_output.scheduler_step,
|
||||
input_batch,
|
||||
self.req_states.draft_tokens[input_batch.idx_mapping],
|
||||
)
|
||||
|
||||
fast_prefill = None
|
||||
if self.fast_prefill is not None:
|
||||
@@ -1529,6 +1641,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
else None
|
||||
),
|
||||
)
|
||||
invalid_draft_positions = None
|
||||
if self.pcp_manager is not None:
|
||||
input_batch = self.pcp_manager.partition_batch(input_batch, batch_desc)
|
||||
return input_batch
|
||||
@@ -1611,6 +1724,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
grammar_output.structured_output_request_ids,
|
||||
grammar_output.num_acceptable_drafts,
|
||||
)
|
||||
invalid_draft_positions = grammar_invalid_draft_positions(
|
||||
input_batch,
|
||||
grammar_output.structured_output_request_ids,
|
||||
grammar_output.num_acceptable_drafts,
|
||||
self.device,
|
||||
)
|
||||
|
||||
sampler_output: SamplerOutput | None
|
||||
if input_batch.num_reqs == 0:
|
||||
@@ -1629,6 +1748,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
input_batch,
|
||||
# Draft logits are needed for probabilistic rejection sampling.
|
||||
self.speculator.draft_logits,
|
||||
invalid_draft_positions,
|
||||
invalid_drafts,
|
||||
)
|
||||
|
||||
@@ -1695,6 +1815,45 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.ec_connector.no_forward(scheduler_output).ec_connector_output,
|
||||
)
|
||||
|
||||
def _prepare_ple_forward(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
batch_desc: BatchExecutionDescriptor,
|
||||
dummy_run: bool,
|
||||
*,
|
||||
serving: bool = False,
|
||||
transaction_id: str | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
connector = self._ple_offload_connector
|
||||
assert connector is not None
|
||||
request_num_tokens = int(input_batch.num_tokens_after_padding)
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
manager = self.cudagraph_manager
|
||||
if manager is None:
|
||||
raise RuntimeError("PLE FULL replay has no CUDA graph manager")
|
||||
copy_num_tokens = manager.ple_copy_rows(batch_desc)
|
||||
else:
|
||||
# PIECEWISE/eager execute the Python op with this actual padded view.
|
||||
copy_num_tokens = int(input_batch.input_ids.shape[0])
|
||||
|
||||
if request_num_tokens != copy_num_tokens:
|
||||
raise RuntimeError(
|
||||
"PLE request/selected-forward copy-count mismatch: "
|
||||
f"request={request_num_tokens}, copy={copy_num_tokens}, "
|
||||
f"mode={batch_desc.cg_mode}"
|
||||
)
|
||||
connector.prepare_forward(
|
||||
input_batch.num_reqs,
|
||||
request_num_tokens,
|
||||
dummy_run,
|
||||
consumer_num_tokens=copy_num_tokens,
|
||||
serving=serving,
|
||||
transaction_id=transaction_id,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
|
||||
@fail_ple_execute_model
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
self,
|
||||
@@ -1708,6 +1867,22 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
randomize_inputs: bool = False,
|
||||
) -> ModelRunnerOutput | IntermediateTensors | None:
|
||||
if not dummy_run:
|
||||
# Bounds-guard instrumentation: raise on the engine thread if a bounds guard fired.
|
||||
# A Python bool; adds no device sync to the step. Late and
|
||||
# rank-local -- see guard_telemetry.check_pending.
|
||||
guard_telemetry.check_pending()
|
||||
# Post-step read of the top-k exact-restart counters, i.e.
|
||||
# everything the previous steps accumulated. Off unless
|
||||
# MERLIN_TOPK_EXACT_COUNTERS_LOG > 0, and then a clock read on
|
||||
# every step but the one per cadence that is actually due. It sits
|
||||
# here rather than at the tail of sample_tokens because that method
|
||||
# returns early on non-last PP ranks, so a tail hook would silently
|
||||
# never fire on them. Outside any capture: this is the engine
|
||||
# thread between steps (read() re-checks).
|
||||
topk_counters.maybe_log()
|
||||
# v2.2: the e2-toggles line if arm() had to defer it (a C++ static
|
||||
# still uninitialised). One bool check once it has been emitted.
|
||||
e2_toggles.maybe_emit()
|
||||
# Update the request states.
|
||||
self.update_pp_decode_requests()
|
||||
self.finish_requests(scheduler_output)
|
||||
@@ -1972,6 +2147,54 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
self.step_timing.forward_start()
|
||||
|
||||
# prepare_inputs has finalized MRV2's GPU buffers. Stage D2H on this
|
||||
# stream before the PLE placeholder can wait for CPU output.
|
||||
connector = self._ple_offload_connector
|
||||
ple_serving = False
|
||||
ple_sequence = None
|
||||
ple_transaction = None
|
||||
ple_correlation = None
|
||||
if connector is not None:
|
||||
ple_serving = not dummy_run and not is_profile
|
||||
ple_sequence = connector.begin_forward(
|
||||
dummy_run=dummy_run,
|
||||
num_tokens=int(input_batch.num_tokens_after_padding),
|
||||
)
|
||||
if _FAULT is not None and ple_serving:
|
||||
ple_correlation = f"sequence:{ple_sequence}"
|
||||
ple_transaction = connector.fault_transaction(ple_sequence)
|
||||
if _FAULT is not None and _FAULT.fires(
|
||||
"exc_before_prepare",
|
||||
role="gpu",
|
||||
unit="runner-forward",
|
||||
eligible=ple_serving,
|
||||
tp_rank=connector.tp_rank,
|
||||
pp_rank=connector.pp_rank,
|
||||
dp_rank=connector.dp_rank,
|
||||
transaction_id=ple_transaction,
|
||||
correlation_id=ple_correlation,
|
||||
):
|
||||
raise RuntimeError("PLE test fault: exc_before_prepare")
|
||||
self._prepare_ple_forward(
|
||||
input_batch,
|
||||
batch_desc,
|
||||
dummy_run,
|
||||
serving=ple_serving,
|
||||
transaction_id=ple_transaction,
|
||||
correlation_id=ple_correlation,
|
||||
)
|
||||
if _FAULT is not None and _FAULT.fires(
|
||||
"exc_after_prepare",
|
||||
role="gpu",
|
||||
unit="runner-forward",
|
||||
eligible=ple_serving,
|
||||
tp_rank=connector.tp_rank,
|
||||
pp_rank=connector.pp_rank,
|
||||
dp_rank=connector.dp_rank,
|
||||
transaction_id=ple_transaction,
|
||||
correlation_id=ple_correlation,
|
||||
):
|
||||
raise RuntimeError("PLE test fault: exc_after_prepare")
|
||||
connector_kwargs = dict(
|
||||
scheduler_output=scheduler_output,
|
||||
request_state_indices=input_batch.idx_mapping,
|
||||
@@ -2029,6 +2252,26 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Eager (NONE): call the raw model directly.
|
||||
model_output = self.model(**model_inputs)
|
||||
|
||||
# The model has consumed every PLE output, so the CPU worker may reuse
|
||||
# the registered buffers for the next request.
|
||||
if connector is not None:
|
||||
connector.release_outputs(
|
||||
serving=ple_serving,
|
||||
transaction_id=ple_transaction,
|
||||
correlation_id=ple_correlation,
|
||||
)
|
||||
if _FAULT is not None and _FAULT.fires(
|
||||
"exc_after_release",
|
||||
role="gpu",
|
||||
unit="runner-forward",
|
||||
eligible=ple_serving,
|
||||
tp_rank=connector.tp_rank,
|
||||
pp_rank=connector.pp_rank,
|
||||
dp_rank=connector.dp_rank,
|
||||
transaction_id=ple_transaction,
|
||||
correlation_id=ple_correlation,
|
||||
):
|
||||
raise RuntimeError("PLE test fault: exc_after_release")
|
||||
self.kv_connector.finish_forward()
|
||||
|
||||
if self.is_last_pp_rank:
|
||||
@@ -2057,6 +2300,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
finished_req_ids=finished_req_ids,
|
||||
ec_connector_output=ec_connector_output,
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
ple_serving=ple_serving,
|
||||
ple_sequence=ple_sequence,
|
||||
ple_transaction=ple_transaction,
|
||||
)
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
@@ -2068,6 +2314,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
return None
|
||||
|
||||
@fail_ple_execute_model
|
||||
@torch.inference_mode()
|
||||
@step_eplb_after()
|
||||
def sample_tokens(
|
||||
@@ -2086,6 +2333,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||
ec_connector_output = self.execute_model_state.ec_connector_output
|
||||
cudagraph_stats = self.execute_model_state.cudagraph_stats
|
||||
ple_serving = self.execute_model_state.ple_serving
|
||||
ple_sequence = self.execute_model_state.ple_sequence
|
||||
ple_transaction = self.execute_model_state.ple_transaction
|
||||
self.execute_model_state = None
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
@@ -2095,7 +2345,33 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if self.pcp_manager is not None:
|
||||
input_batch = self.pcp_manager.global_batch
|
||||
assert self.pp_handler is not None
|
||||
connector = self._ple_offload_connector
|
||||
if (
|
||||
connector is not None
|
||||
and _FAULT is not None
|
||||
and ple_serving
|
||||
and ple_sequence is not None
|
||||
and connector.completion_is_pending(ple_sequence)
|
||||
and _FAULT.fires(
|
||||
"exc_async_output",
|
||||
role="gpu",
|
||||
unit="runner-output-relay",
|
||||
eligible=True,
|
||||
tp_rank=connector.tp_rank,
|
||||
pp_rank=connector.pp_rank,
|
||||
dp_rank=connector.dp_rank,
|
||||
transaction_id=ple_transaction,
|
||||
correlation_id=f"sequence:{ple_sequence}",
|
||||
)
|
||||
):
|
||||
raise RuntimeError("PLE test fault: exc_async_output")
|
||||
all_decode_next = self.pp_handler.receive(input_batch)
|
||||
# The accepted counts of this step arrive pp_size steps later; keep
|
||||
# the state their postprocess needs with this step's pending entry
|
||||
# (before the immediate prefill-chunk postprocess below).
|
||||
self.pp_handler.attach_deferred_state(
|
||||
self.model_state.detach_deferred_state()
|
||||
)
|
||||
# Optimistically update num_computed_tokens for entire batch here.
|
||||
# Will be adjusted for rejections if necessary in update_requests.
|
||||
self.postprocess_num_computed_tokens(input_batch)
|
||||
@@ -2265,8 +2541,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
return async_output
|
||||
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
return self.draft_tokens_handler.get_draft_tokens()
|
||||
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||
return self.draft_tokens_handler.get_draft_tokens(step_id=step_id)
|
||||
|
||||
@torch.inference_mode()
|
||||
@step_eplb_after()
|
||||
@@ -2308,6 +2584,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
main_stream=self.main_stream,
|
||||
copy_stream=self.output_copy_stream,
|
||||
)
|
||||
# The last window of a run is the one a cadence log always
|
||||
# misses; emit it before the device state goes away.
|
||||
topk_counters.maybe_log(force=True)
|
||||
|
||||
self.postprocess_num_computed_tokens(input_batch)
|
||||
return async_output
|
||||
@@ -2324,6 +2603,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
"""Release GPU tensors (model weights, KV caches, workspace) so that
|
||||
memory is reclaimable when running in the same process."""
|
||||
torch.accelerator.synchronize()
|
||||
# The last window of a run is the one a cadence log always
|
||||
# misses; emit it before the device state goes away.
|
||||
topk_counters.maybe_log(force=True)
|
||||
# v2.2: a still-deferred e2-toggles line goes out now, 'uninit' and all.
|
||||
e2_toggles.maybe_emit(force=True)
|
||||
if self._ple_offload_connector is not None:
|
||||
self._ple_offload_connector.close()
|
||||
self._ple_offload_connector = None
|
||||
if self.aux_output_connector is not None:
|
||||
self.aux_output_connector.close()
|
||||
set_offloader(None)
|
||||
@@ -2396,6 +2683,9 @@ class ExecuteModelState(NamedTuple):
|
||||
finished_req_ids: set[str]
|
||||
ec_connector_output: ECConnectorOutput | None
|
||||
cudagraph_stats: CUDAGraphStat | None
|
||||
ple_serving: bool
|
||||
ple_sequence: int | None
|
||||
ple_transaction: str | None
|
||||
|
||||
|
||||
class BatchReqState(NamedTuple):
|
||||
|
||||
@@ -126,6 +126,14 @@ class ModelState(ABC):
|
||||
def apply_staged_writes(self) -> None:
|
||||
return None
|
||||
|
||||
def set_block_tables(self, block_tables: Any) -> None:
|
||||
"""Register the runner's `BlockTables` once it exists.
|
||||
|
||||
Model states that index a block table outside the step that gathered it
|
||||
(mamba "align" + the PP-deferred post-process) need the persistent
|
||||
per-req-slot tables, not the batch-order gather. No-op by default."""
|
||||
return None
|
||||
|
||||
def get_additional_cg_support(self) -> tuple[AttentionCGSupport, str | None]:
|
||||
"""Cudagraph support of attention groups this ModelState builds outside
|
||||
``init_attn_backend`` (e.g. encoder-only layers).
|
||||
@@ -155,6 +163,19 @@ class ModelState(ABC):
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
def detach_deferred_state(self) -> Any:
|
||||
"""Non-last PP rank, after this step's forward: hand over any state the
|
||||
step's postprocess needs once its sampled tokens arrive (pp_size steps
|
||||
later). The PP handler keeps it with the step's pending entry. None by
|
||||
default (nothing deferred)."""
|
||||
return None
|
||||
|
||||
def restore_deferred_state(self, state: Any) -> None:
|
||||
"""Counterpart of `detach_deferred_state`, called right before the
|
||||
deferred postprocess of the step that detached `state`."""
|
||||
if state is not None:
|
||||
raise RuntimeError(f"{type(self).__name__} cannot restore {state!r}")
|
||||
|
||||
@abstractmethod
|
||||
def prepare_inputs_embeds(
|
||||
self,
|
||||
|
||||
@@ -90,6 +90,9 @@ class MambaHybridModelState(DefaultModelState):
|
||||
# kernel reusing the postprocess copy machinery, so the per-step src
|
||||
# columns and the running state_idx are kept GPU-resident.
|
||||
self._align_mode = self.cache_config.mamba_cache_mode == "align"
|
||||
# Set by the runner via `set_block_tables`; the align post-process needs
|
||||
# the persistent per-req-slot tables, not the per-step gather.
|
||||
self._runner_block_tables: Any = None
|
||||
self.recoverssm = (
|
||||
RecoverSSMState() if self.cache_config.use_recoverssm else None
|
||||
)
|
||||
@@ -108,6 +111,9 @@ class MambaHybridModelState(DefaultModelState):
|
||||
self._mamba_spec: MambaSpec | None = None
|
||||
self._mamba_state_copy_funcs: MambaStateCopyFuncsByType | None = None
|
||||
|
||||
def set_block_tables(self, block_tables: Any) -> None:
|
||||
self._runner_block_tables = block_tables
|
||||
|
||||
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
|
||||
super().add_request(req_index, new_req_data)
|
||||
# Must reset the speculative acceptance count in this idx which could be stale.
|
||||
@@ -168,11 +174,24 @@ class MambaHybridModelState(DefaultModelState):
|
||||
# block_tables are batch-order slices of the persistent
|
||||
# input_block_tables (stable data_ptr), so the metadata is captured
|
||||
# once here and reused across steps.
|
||||
#
|
||||
# Also hand over the per-req-slot tables the gather reads from
|
||||
# (block_table.py:53-59). The align post-process can run pp_size
|
||||
# steps after the gather that produced its idx_mapping, at which
|
||||
# point the batch-order buffer belongs to another micro-batch; it
|
||||
# indexes these by req slot instead.
|
||||
persistent_block_tables = None
|
||||
if self._runner_block_tables is not None:
|
||||
persistent_block_tables = [
|
||||
self._runner_block_tables.block_tables[gid].gpu
|
||||
for gid in mamba_group_ids
|
||||
]
|
||||
ctx.initialize_from_forward_context(
|
||||
kv_cache_config,
|
||||
forward_context,
|
||||
copy_funcs,
|
||||
[block_tables[gid] for gid in mamba_group_ids],
|
||||
persistent_block_tables=persistent_block_tables,
|
||||
)
|
||||
return ctx
|
||||
|
||||
@@ -339,6 +358,20 @@ class MambaHybridModelState(DefaultModelState):
|
||||
)
|
||||
return attn_metadata
|
||||
|
||||
def detach_deferred_state(self) -> Any:
|
||||
# Non-last PP rank: the RecoverSSM commit of this step has to wait for
|
||||
# the accepted counts of the PP broadcast; snapshot its inputs now.
|
||||
if self.recoverssm is None:
|
||||
return None
|
||||
return self.recoverssm.detach_step()
|
||||
|
||||
def restore_deferred_state(self, state: Any) -> None:
|
||||
if state is None:
|
||||
return
|
||||
if self.recoverssm is None:
|
||||
raise RuntimeError("deferred RecoverSSM state without RecoverSSM")
|
||||
self.recoverssm.restore_step(state)
|
||||
|
||||
def postprocess_state(
|
||||
self,
|
||||
idx_mapping: torch.Tensor,
|
||||
|
||||
@@ -1,20 +1,77 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.recoverssm_metadata import RecoverSSMMetadata
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DeferredRecoverSSMStep:
|
||||
"""One verify step's RecoverSSM commits, detached on a non-last PP rank.
|
||||
|
||||
The rank verified the window at step T but receives the accepted counts
|
||||
only when the PP broadcast of step T is consumed (pp_size steps later).
|
||||
The metadata is a snapshot (`snapshot_for_deferred_commit`), so the steps
|
||||
in between cannot overwrite its commit inputs. It rides on the PP
|
||||
handler's pending entry for step T and is restored right before that
|
||||
entry's postprocess."""
|
||||
|
||||
metadata: tuple[RecoverSSMMetadata, ...]
|
||||
|
||||
|
||||
class RecoverSSMState:
|
||||
"""Coordinates RecoverSSM metadata between attention and postprocessing."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._step: tuple[RecoverSSMMetadata, ...] | None = None
|
||||
# True while `_step` is a restored PP-deferred step: its commit must
|
||||
# skip rows whose request was freed since the verify (their state
|
||||
# blocks may already belong to another request).
|
||||
self._step_is_deferred = False
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Drop the recorded step. Its metadata holds the commit contexts, which
|
||||
reference the KV cache tensors they were built for."""
|
||||
self._step = None
|
||||
self._step_is_deferred = False
|
||||
|
||||
def detach_step(self) -> DeferredRecoverSSMStep | None:
|
||||
"""Take this step's commits for a deferred (non-last PP rank) commit.
|
||||
|
||||
Returns None when the step has nothing to commit (no spec rows)."""
|
||||
step = self._step
|
||||
self._step = None
|
||||
self._step_is_deferred = False
|
||||
if step is None:
|
||||
return None
|
||||
live = tuple(m for m in step if m.has_recoverssm_commit())
|
||||
if not live:
|
||||
return None
|
||||
return DeferredRecoverSSMStep(
|
||||
tuple(m.snapshot_for_deferred_commit() for m in live)
|
||||
)
|
||||
|
||||
def restore_step(self, deferred: DeferredRecoverSSMStep | None) -> None:
|
||||
"""Make a detached step the one the next `commit_step` commits."""
|
||||
if self._step is not None:
|
||||
# Every non-last-rank step detaches its record in sample_tokens, so
|
||||
# only a capture-time record (for_capture=False under PIECEWISE)
|
||||
# can be here; it has no sampled tokens and is never committed.
|
||||
logger.warning_once(
|
||||
"RecoverSSM: dropping a recorded step that was never committed "
|
||||
"(expected only after a PIECEWISE capture)"
|
||||
)
|
||||
self._step = None if deferred is None else deferred.metadata
|
||||
self._step_is_deferred = deferred is not None
|
||||
|
||||
def record_step(
|
||||
self,
|
||||
@@ -44,10 +101,26 @@ class RecoverSSMState:
|
||||
num_accepted_tokens: torch.Tensor,
|
||||
) -> None:
|
||||
step = self._step
|
||||
deferred = self._step_is_deferred
|
||||
self._step = None
|
||||
self._step_is_deferred = False
|
||||
if isinstance(num_sampled, int) or step is None:
|
||||
if deferred and step is not None:
|
||||
raise RuntimeError(
|
||||
"RecoverSSM: a deferred PP commit was restored without "
|
||||
"sampled counts"
|
||||
)
|
||||
return
|
||||
|
||||
if deferred:
|
||||
# Under PP the consume filter marks freed/non-sampling rows with -1
|
||||
# in idx_mapping. A zero count makes the commit plan skip the row
|
||||
# (commit_len 0): no state, boundary or conv write into blocks the
|
||||
# request no longer owns.
|
||||
num_sampled = torch.where(
|
||||
idx_mapping >= 0, num_sampled, torch.zeros_like(num_sampled)
|
||||
)
|
||||
|
||||
for metadata in step:
|
||||
postprocess_meta = metadata.commit_recoverssm_state(num_sampled)
|
||||
if postprocess_meta is None:
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""Pipeline Parallelism utils for V2 Model Runner."""
|
||||
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -33,6 +33,9 @@ class PendingRecv:
|
||||
# detect requests aborted since then.
|
||||
gen_at_receive_np: np.ndarray # [num_reqs]
|
||||
draft_tokens: torch.Tensor | None = None # [num_reqs, num_speculative_steps]
|
||||
# Model-state data the step's postprocess needs once the sampled tokens
|
||||
# arrive (RecoverSSM: the snapshot of the step's deferred commits).
|
||||
deferred_state: object | None = None
|
||||
|
||||
|
||||
def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
|
||||
@@ -167,8 +170,34 @@ class PPHandler:
|
||||
num_sampled=slot.num_sampled,
|
||||
num_rejected=slot.num_rejected,
|
||||
idx_mapping=idx_mapping,
|
||||
deferred_state=slot.deferred_state,
|
||||
)
|
||||
|
||||
def attach_deferred_state(self, state: object | None) -> None:
|
||||
"""Attach model-state data to the entry `receive` just reserved for this
|
||||
step. Called right after `receive` in the same step."""
|
||||
assert not self.is_last_rank
|
||||
if state is None:
|
||||
return
|
||||
slot = self.queue[-1]
|
||||
if slot is None:
|
||||
# receive() skipped the broadcast: no row of this step samples, so
|
||||
# no row can have a speculative window to commit.
|
||||
raise RuntimeError(
|
||||
"deferred model state for a step without sampled outputs"
|
||||
)
|
||||
self.queue[-1] = replace(slot, deferred_state=state)
|
||||
|
||||
def drop_deferred_states(self) -> int:
|
||||
"""Release every pending entry's deferred state (profiling teardown: the
|
||||
state references the KV cache it was built for). Returns the count."""
|
||||
dropped = 0
|
||||
for i, slot in enumerate(self.queue):
|
||||
if slot is not None and slot.deferred_state is not None:
|
||||
self.queue[i] = replace(slot, deferred_state=None)
|
||||
dropped += 1
|
||||
return dropped
|
||||
|
||||
def broadcast_drafts(
|
||||
self, draft_tokens: torch.Tensor, input_batch: InputBatch
|
||||
) -> None:
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
@@ -10,6 +13,10 @@ from vllm.v1.outputs import DraftTokenIds
|
||||
from vllm.v1.worker.gpu.async_utils import async_copy_to_np
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
|
||||
# Keep enough consumption-time snapshots to cover async PP in-flight
|
||||
# batches (`max_concurrent_batches` is `pp_size + 1`) with headroom.
|
||||
_DEFAULT_MAX_SNAPSHOTS = 8
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -26,48 +33,135 @@ def get_pp_safe_draft_load_config(load_config: LoadConfig) -> LoadConfig:
|
||||
return load_config
|
||||
|
||||
|
||||
class DraftTokensHandler:
|
||||
def __init__(self, device: torch.device | None = None):
|
||||
self.device = device
|
||||
self.copy_stream = torch.cuda.Stream(device)
|
||||
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
|
||||
self.copy_event = torch.cuda.Event(blocking=True)
|
||||
@dataclass
|
||||
class _DraftSnapshot:
|
||||
step_id: int | None
|
||||
req_ids: list[str]
|
||||
num_draft_tokens: int
|
||||
draft_tokens_np: np.ndarray | None
|
||||
copy_event: torch.cuda.Event | None
|
||||
|
||||
self.req_ids: list[str] = []
|
||||
self.draft_tokens_np: np.ndarray | None = None
|
||||
self.num_draft_tokens: int = 0
|
||||
def to_draft_token_ids(self) -> DraftTokenIds:
|
||||
if self.draft_tokens_np is not None:
|
||||
if self.copy_event is not None:
|
||||
self.copy_event.synchronize()
|
||||
draft_token_ids = self.draft_tokens_np.tolist()
|
||||
else:
|
||||
draft_token_ids = [[-1] * self.num_draft_tokens for _ in self.req_ids]
|
||||
return DraftTokenIds(self.req_ids, draft_token_ids)
|
||||
|
||||
|
||||
class DraftTokensHandler:
|
||||
def __init__(
|
||||
self,
|
||||
device: torch.device | None = None,
|
||||
max_snapshots: int = _DEFAULT_MAX_SNAPSHOTS,
|
||||
):
|
||||
self.device = device
|
||||
self.copy_stream: torch.cuda.Stream | None = None
|
||||
if device is not None and device.type == "cuda":
|
||||
self.copy_stream = torch.cuda.Stream(device)
|
||||
|
||||
self._max_snapshots = max(1, max_snapshots)
|
||||
# Consumption-time snapshots tagged with scheduler step identity.
|
||||
self._consumed: deque[_DraftSnapshot] = deque(maxlen=self._max_snapshots)
|
||||
# Latest proposal-time drafts for the non-async post_step path.
|
||||
self._proposed: _DraftSnapshot | None = None
|
||||
|
||||
def _copy_drafts(
|
||||
self, draft_tokens: torch.Tensor
|
||||
) -> tuple[np.ndarray, torch.cuda.Event | None]:
|
||||
if self.copy_stream is None:
|
||||
return draft_tokens.detach().cpu().numpy(), None
|
||||
|
||||
current_stream = torch.cuda.current_stream(self.device)
|
||||
self.copy_stream.wait_stream(current_stream)
|
||||
copy_event = torch.cuda.Event(blocking=True)
|
||||
with torch.cuda.stream(self.copy_stream):
|
||||
draft_tokens_np = async_copy_to_np(draft_tokens)
|
||||
# draft_tokens is a temporary allocation on the main stream and
|
||||
# read here on copy_stream; without record_stream, the caching
|
||||
# allocator may reuse its memory before the async copy executes.
|
||||
draft_tokens.record_stream(self.copy_stream)
|
||||
copy_event.record()
|
||||
return draft_tokens_np, copy_event
|
||||
|
||||
def snapshot_consumed_drafts(
|
||||
self,
|
||||
step_id: int,
|
||||
input_batch: InputBatch,
|
||||
draft_tokens: torch.Tensor,
|
||||
) -> None:
|
||||
"""Record drafts that prepare_inputs fed into input_ids for this step.
|
||||
|
||||
Tagged with the scheduler step so the engine can retrieve the matching
|
||||
snapshot later by equality. Batches without structured-output requests
|
||||
are not stored, so they cannot evict a useful snapshot.
|
||||
"""
|
||||
if not input_batch.has_structured_output_reqs:
|
||||
return
|
||||
|
||||
draft_tokens_np, copy_event = self._copy_drafts(draft_tokens)
|
||||
self._consumed.append(
|
||||
_DraftSnapshot(
|
||||
step_id=step_id,
|
||||
req_ids=list(input_batch.req_ids),
|
||||
num_draft_tokens=draft_tokens.shape[1],
|
||||
draft_tokens_np=draft_tokens_np,
|
||||
copy_event=copy_event,
|
||||
)
|
||||
)
|
||||
|
||||
def set_draft_tokens(
|
||||
self, input_batch: InputBatch, draft_tokens: torch.Tensor
|
||||
) -> None:
|
||||
self.req_ids = input_batch.req_ids
|
||||
self.num_draft_tokens = draft_tokens.shape[1]
|
||||
req_ids = list(input_batch.req_ids)
|
||||
num_draft_tokens = draft_tokens.shape[1]
|
||||
if not input_batch.has_structured_output_reqs:
|
||||
# No draft token validation needs to be performed by
|
||||
# the scheduler for this batch.
|
||||
self.draft_tokens_np = None
|
||||
self._proposed = _DraftSnapshot(
|
||||
step_id=None,
|
||||
req_ids=req_ids,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
draft_tokens_np=None,
|
||||
copy_event=None,
|
||||
)
|
||||
return
|
||||
|
||||
# For spec decoding + structured outputs, we must transfer the
|
||||
# draft tokens back to the scheduler for grammar validation.
|
||||
current_stream = torch.cuda.current_stream(self.device)
|
||||
self.copy_stream.wait_stream(current_stream)
|
||||
with torch.cuda.stream(self.copy_stream):
|
||||
self.draft_tokens_np = async_copy_to_np(draft_tokens)
|
||||
# draft_tokens is a temporary allocation on the main stream and read here on
|
||||
# copy_stream; without record_stream, the caching allocator may reuse its
|
||||
# memory before the async copy executes.
|
||||
draft_tokens.record_stream(self.copy_stream)
|
||||
self.copy_event.record()
|
||||
draft_tokens_np, copy_event = self._copy_drafts(draft_tokens)
|
||||
self._proposed = _DraftSnapshot(
|
||||
step_id=None,
|
||||
req_ids=req_ids,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
draft_tokens_np=draft_tokens_np,
|
||||
copy_event=copy_event,
|
||||
)
|
||||
|
||||
def get_draft_tokens(self) -> DraftTokenIds | None:
|
||||
if self.draft_tokens_np is not None:
|
||||
self.copy_event.synchronize()
|
||||
draft_token_ids = self.draft_tokens_np.tolist()
|
||||
else:
|
||||
# This case only happens when async scheduling is disabled.
|
||||
draft_token_ids = [[-1] * self.num_draft_tokens for _ in self.req_ids]
|
||||
return DraftTokenIds(self.req_ids, draft_token_ids)
|
||||
def get_draft_tokens(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||
"""Return draft tokens for grammar validation.
|
||||
|
||||
If ``step_id`` is set, look up the consumption-time snapshot for that
|
||||
scheduler step by equality. A miss returns None so placeholders stay
|
||||
-1 and fail-closed invalidation can pin acceptance via is_valid_draft.
|
||||
If ``step_id`` is None, return the latest proposal-time drafts (the
|
||||
non-async post_step path).
|
||||
"""
|
||||
if step_id is not None:
|
||||
for snapshot in reversed(self._consumed):
|
||||
if snapshot.step_id == step_id:
|
||||
return snapshot.to_draft_token_ids()
|
||||
logger.debug(
|
||||
"Draft-token snapshot miss for scheduler step %s",
|
||||
step_id,
|
||||
)
|
||||
return None
|
||||
|
||||
if self._proposed is None:
|
||||
return DraftTokenIds([], [])
|
||||
return self._proposed.to_draft_token_ids()
|
||||
|
||||
|
||||
def get_parallel_drafting_token_id(hf_config) -> int:
|
||||
|
||||
@@ -214,6 +214,7 @@ from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
|
||||
from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper
|
||||
from vllm.v1.worker.kv_connector_model_runner_mixin import KVConnectorModelRunnerMixin
|
||||
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
|
||||
from vllm.v1.worker.ring_tags import clear_ring_row_tags, ring_tag_layers
|
||||
from vllm.v1.worker.ubatch_utils import (
|
||||
UBatchSlices,
|
||||
check_ubatch_thresholds,
|
||||
@@ -1148,6 +1149,7 @@ class GPUModelRunner(
|
||||
runner_only_attn_layers=self.runner_only_attn_layers,
|
||||
static_forward_context=self.compilation_config.static_forward_context,
|
||||
num_blocks=self.kv_cache_config.num_blocks,
|
||||
skip_kv_cache_group_ids=self.kv_cache_config.slot_packed_group_ids,
|
||||
)
|
||||
|
||||
def _zero_block_ids(self, block_ids: list[int]) -> None:
|
||||
@@ -1218,6 +1220,14 @@ class GPUModelRunner(
|
||||
# stale NaN/data from corrupting attention or SSM computation.
|
||||
if scheduler_output.new_block_ids_to_zero:
|
||||
self._zero_block_ids(scheduler_output.new_block_ids_to_zero)
|
||||
# v2.2 ring-row tag fix: clear the tag words of newly claimed ring blocks. Not
|
||||
# under the zeroing gate; after the zeroer, before copies and forward.
|
||||
if scheduler_output.new_ring_block_ids_to_clear:
|
||||
clear_ring_row_tags(
|
||||
ring_tag_layers(self.compilation_config.static_forward_context),
|
||||
scheduler_output.new_ring_block_ids_to_clear,
|
||||
self.device,
|
||||
)
|
||||
if scheduler_output.kv_cache_block_copies:
|
||||
copy_kv_cache_blocks_inplace(
|
||||
self.kv_caches,
|
||||
@@ -4821,7 +4831,8 @@ class GPUModelRunner(
|
||||
bt.block_table.gpu.copy_(saved_gpu)
|
||||
bt.block_table.cpu.copy_(saved_cpu)
|
||||
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||
del step_id # V2-only identity; V1 keeps a single proposal slot.
|
||||
if not self.num_spec_tokens or not self._draft_token_req_ids:
|
||||
return None
|
||||
draft_token_ids, req_ids = self._get_draft_token_ids_cpu()
|
||||
@@ -7121,6 +7132,7 @@ class GPUModelRunner(
|
||||
kv_cache_config=self.kv_cache_config,
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
is_profiling=is_profiling,
|
||||
use_replayssm=self.cache_config.use_replayssm,
|
||||
)
|
||||
# Trigger cudagraph dispatching keys initialization after
|
||||
# resolved cudagraph mode.
|
||||
|
||||
@@ -91,7 +91,10 @@ from vllm.v1.outputs import (
|
||||
DraftTokenIds,
|
||||
ModelRunnerOutput,
|
||||
)
|
||||
from vllm.v1.ple_offload.fatal import fail_ple_execute_model
|
||||
from vllm.v1.utils import compute_iteration_details, report_usage_stats
|
||||
from vllm.v1.worker import e2_memstats as _e2_memstats
|
||||
from vllm.v1.worker import guard_telemetry
|
||||
from vllm.v1.worker.sentinel.gpu_worker_sentinel import WorkerSentinel
|
||||
from vllm.v1.worker.startup_plan import (
|
||||
maybe_apply_startup_plan,
|
||||
@@ -230,6 +233,22 @@ class Worker(WorkerBase):
|
||||
validate_worker_profiler_config(self.profiler_config)
|
||||
|
||||
self.use_v2_model_runner = vllm_config.use_v2_model_runner
|
||||
# PLE pull transport (VLLM_E2_PLE_PULL_TRANSPORT). Off: both stay
|
||||
# None/False and every PLE hook in this class is a no-op.
|
||||
self._ple_offload_worker_handle: Any | None = None
|
||||
self._ple_offload_stopping = False
|
||||
self._ple_offload_enabled = self._has_ple_layers()
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
if self._ple_offload_enabled:
|
||||
self._validate_ple_offload_config()
|
||||
elif self.rank == 0 and self.parallel_config.data_parallel_rank == 0:
|
||||
text_config = self.model_config.hf_text_config
|
||||
logger.warning(
|
||||
"VLLM_E2_PLE_PULL_TRANSPORT is enabled, but this rank owns "
|
||||
"no PLE layer (ple_layer_ids=%s); no PLE offload connector "
|
||||
"is created here.",
|
||||
getattr(text_config, "ple_layer_ids", None),
|
||||
)
|
||||
|
||||
# Device handles of the previous step's PP intermediate-tensor send.
|
||||
self._pp_send_work: list[Handle] = []
|
||||
@@ -237,6 +256,148 @@ class Worker(WorkerBase):
|
||||
# Resolved lazily on first sleep/wake; persists worker-process state.
|
||||
self._sleep_mode_backend: SleepModeBackend | None = None
|
||||
|
||||
def _has_ple_layers(self) -> bool:
|
||||
"""Return whether this model configuration constructs PLE layers."""
|
||||
if not envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
return False
|
||||
text_config = self.model_config.hf_text_config
|
||||
ple_layer_ids = getattr(text_config, "ple_layer_ids", None)
|
||||
if not ple_layer_ids:
|
||||
return False
|
||||
# PP (vllm#54709, carried from the earlier overlay): only the pipeline rank
|
||||
# that owns a PLE layer takes part in the offload (rank 0 with the
|
||||
# default partition). Other ranks build PPMissingLayer there and must
|
||||
# skip the connector entirely.
|
||||
pp_size = self.parallel_config.pipeline_parallel_size
|
||||
if pp_size > 1:
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
|
||||
tp_size = self.parallel_config.tensor_parallel_size
|
||||
pp_rank = (self.rank // tp_size) % pp_size
|
||||
start, end = get_pp_indices(
|
||||
int(text_config.num_hidden_layers), pp_rank, pp_size
|
||||
)
|
||||
return any(start <= int(i) - 1 < end for i in ple_layer_ids)
|
||||
return True
|
||||
|
||||
def _validate_ple_offload_config(self) -> None:
|
||||
"""Reject unsupported PLE offload execution modes."""
|
||||
parallel_config = self.parallel_config
|
||||
unsupported = []
|
||||
if not self.use_v2_model_runner:
|
||||
unsupported.append("model runner V1")
|
||||
if not current_platform.is_cuda():
|
||||
unsupported.append(f"device={current_platform.device_type}")
|
||||
if parallel_config.nnodes != 1:
|
||||
unsupported.append(f"nnodes={parallel_config.nnodes}")
|
||||
if parallel_config.data_parallel_backend != "mp":
|
||||
unsupported.append(f"DP backend={parallel_config.data_parallel_backend}")
|
||||
if (
|
||||
parallel_config.data_parallel_size_local
|
||||
!= parallel_config.data_parallel_size
|
||||
):
|
||||
unsupported.append(
|
||||
"non-local DP "
|
||||
f"({parallel_config.data_parallel_size_local}/"
|
||||
f"{parallel_config.data_parallel_size} local ranks)"
|
||||
)
|
||||
# PP allowed (vllm#54709): PLE layers must live on pipeline rank 0
|
||||
# (enforced placement-based in models/config.py); num_workers stays
|
||||
# dp_size * tp_size = the ranks of the owning stage.
|
||||
if parallel_config.prefill_context_parallel_size != 1:
|
||||
unsupported.append(f"PCP={parallel_config.prefill_context_parallel_size}")
|
||||
if parallel_config.decode_context_parallel_size != 1:
|
||||
unsupported.append(f"DCP={parallel_config.decode_context_parallel_size}")
|
||||
if parallel_config.use_ubatching:
|
||||
unsupported.append("ubatching/DBO")
|
||||
if self.model_config.architecture not in {
|
||||
"Qwen4ExpForCausalLM",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
}:
|
||||
unsupported.append("architecture")
|
||||
if self.vllm_config.weight_transfer_config is not None:
|
||||
unsupported.append("weight transfer")
|
||||
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
"VLLM_E2_PLE_PULL_TRANSPORT does not support the requested "
|
||||
"configuration. Unsupported settings: "
|
||||
f"{', '.join(unsupported)}"
|
||||
)
|
||||
|
||||
def spawn_ple_offload(self) -> None:
|
||||
"""Spawn one node-local PLE CPU worker from DP0/TP0."""
|
||||
if (
|
||||
not self._ple_offload_enabled
|
||||
or self.rank != 0
|
||||
or self.parallel_config.data_parallel_rank != 0
|
||||
):
|
||||
return
|
||||
|
||||
from vllm.v1.ple_offload.worker import PleOffloadWorker
|
||||
|
||||
ipc_addr = self.parallel_config._ple_offload_ipc_path
|
||||
if not ipc_addr:
|
||||
raise RuntimeError("PLE offload IPC address was not initialized")
|
||||
dp_size = self.parallel_config.data_parallel_size
|
||||
tp_size = self.parallel_config.tensor_parallel_size
|
||||
num_workers = dp_size * tp_size
|
||||
logger.info(
|
||||
"PleOffload: spawning worker "
|
||||
"(rank=%d, local_rank=%d, dp_size=%d, tp_size=%d, "
|
||||
"num_workers=%d, ipc_addr=%s).",
|
||||
self.rank,
|
||||
self.local_rank,
|
||||
dp_size,
|
||||
tp_size,
|
||||
num_workers,
|
||||
ipc_addr,
|
||||
)
|
||||
self._ple_offload_worker_handle = PleOffloadWorker.make_process(
|
||||
self.vllm_config,
|
||||
num_workers,
|
||||
ipc_addr,
|
||||
)
|
||||
self._start_ple_offload_watchdog()
|
||||
|
||||
def _start_ple_offload_watchdog(self) -> None:
|
||||
"""Fail this worker if the PLE offload process exits unexpectedly.
|
||||
|
||||
Host-mapped pull: a dead producer would leave the captured forward
|
||||
waiting on its host flag forever; turn that into a fast, visible
|
||||
engine failure instead of a hang. ``shutdown`` sets the stop marker
|
||||
before closing the handle, so a normal close is not a failure.
|
||||
"""
|
||||
import threading
|
||||
|
||||
handle = self._ple_offload_worker_handle
|
||||
self._ple_offload_stopping = False
|
||||
|
||||
def watch() -> None:
|
||||
returncode = handle.proc.wait()
|
||||
if getattr(self, "_ple_offload_stopping", False):
|
||||
return
|
||||
logger.error(
|
||||
"PLE offload process %s exited unexpectedly (exitcode %s); "
|
||||
"failing this worker so the engine does not hang on a PLE wait",
|
||||
handle.proc.pid,
|
||||
returncode,
|
||||
)
|
||||
os._exit(1)
|
||||
|
||||
threading.Thread(
|
||||
target=watch, name="ple-offload-watchdog", daemon=True
|
||||
).start()
|
||||
|
||||
def wait_ple_offload_ready(self) -> None:
|
||||
"""Wait until weights and every TP registration are ready."""
|
||||
if self._ple_offload_worker_handle is None:
|
||||
return
|
||||
|
||||
from vllm.v1.ple_offload.worker import PleOffloadWorker
|
||||
|
||||
PleOffloadWorker.wait_for_ready(self._ple_offload_worker_handle)
|
||||
|
||||
@property
|
||||
def sleep_mode_backend(self) -> "SleepModeBackend":
|
||||
if self._sleep_mode_backend is None:
|
||||
@@ -573,6 +734,19 @@ class Worker(WorkerBase):
|
||||
):
|
||||
self.model_runner.load_model(load_dummy_weights=load_dummy_weights)
|
||||
|
||||
# Model loading constructs the PLE placeholders whose buffers are
|
||||
# registered with the offload process by the model runner V2 connector.
|
||||
if self._ple_offload_enabled:
|
||||
model_runner = cast("GPUModelRunnerV2", self.model_runner)
|
||||
model_runner._setup_ple_offload(self.parallel_config._ple_offload_ipc_path)
|
||||
|
||||
# Log-only: this rank's PLE / capture identity for the target, read
|
||||
# after the offload placeholders are armed (v1/worker/e2_identity.py).
|
||||
# The drafter's line follows initialize_kv_cache (initialize_from_config).
|
||||
from vllm.v1.worker.e2_identity import log_launch_identity
|
||||
|
||||
log_launch_identity(self.model_runner, self.vllm_config)
|
||||
|
||||
if has_ec_transfer():
|
||||
get_ec_transfer().start_worker_services()
|
||||
|
||||
@@ -593,6 +767,10 @@ class Worker(WorkerBase):
|
||||
self.model_runner.update_config(overrides)
|
||||
|
||||
def reload_weights(self, *args, **kwargs) -> None:
|
||||
if self._ple_offload_enabled:
|
||||
raise NotImplementedError(
|
||||
"Weight reload is not supported with the PLE pull transport"
|
||||
)
|
||||
with set_current_vllm_config(self.vllm_config):
|
||||
self.model_runner.reload_weights(*args, **kwargs)
|
||||
|
||||
@@ -857,6 +1035,12 @@ class Worker(WorkerBase):
|
||||
kv_cache_allocation_context=mem_pool_context,
|
||||
)
|
||||
|
||||
# Log-only: the drafter's identity line, now that initialize_kv_cache
|
||||
# has resolved its cudagraph managers (v1/worker/e2_identity.py).
|
||||
from vllm.v1.worker.e2_identity import log_draft_identity
|
||||
|
||||
log_draft_identity(self.model_runner)
|
||||
|
||||
# Build KV-zero metadata outside the CuMem pool so the bookkeeping
|
||||
# GPU tensors (seg_addrs, block-id buffers) use the standard PyTorch
|
||||
# allocator and are not discarded during sleep/wake cycles.
|
||||
@@ -922,6 +1106,13 @@ class Worker(WorkerBase):
|
||||
with self._get_cudagraph_capture_context():
|
||||
cuda_graph_memory_bytes = self.model_runner.capture_model()
|
||||
|
||||
# Log-only: this rank's determinism toggles, once, now that the warmup
|
||||
# and capture above have launched persistent_topk and Marlin MoE, so
|
||||
# their C++ statics are filled (v1/worker/e2_toggles.py). Never raises.
|
||||
from vllm.v1.worker import e2_toggles
|
||||
|
||||
e2_toggles.arm()
|
||||
|
||||
# Compare actual vs estimated CUDA graph memory (if we did profiling)
|
||||
if (
|
||||
hasattr(self, "cudagraph_memory_estimate")
|
||||
@@ -1045,6 +1236,16 @@ class Worker(WorkerBase):
|
||||
# gate so subsequent `execute_model` / `sample_tokens` calls enforce it.
|
||||
enable_gpu_sync_check()
|
||||
|
||||
# Bounds-guard instrumentation: same moment, same reason. The guard counters
|
||||
# were allocated in the model runner constructor, but the thread that READS
|
||||
# them must not exist during capture -- a .tolist() from a second thread
|
||||
# while a capture is active is what cudaStreamCaptureModeGlobal rejects.
|
||||
guard_telemetry.start_reader()
|
||||
|
||||
# Allocator telemetry: armed only after warm-up and CUDA-graph capture (env-gated).
|
||||
if _e2_memstats.enabled():
|
||||
_e2_memstats.ensure_started(self)
|
||||
|
||||
return CompilationTimes(
|
||||
language_model=self.compilation_config.compilation_time,
|
||||
encoder=self.compilation_config.encoder_compilation_time,
|
||||
@@ -1256,6 +1457,7 @@ class Worker(WorkerBase):
|
||||
) -> ModelRunnerOutput | AsyncModelRunnerOutput:
|
||||
return self.model_runner.sample_tokens(grammar_output)
|
||||
|
||||
@fail_ple_execute_model
|
||||
@torch.inference_mode()
|
||||
@with_gpu_sync_check
|
||||
def execute_model(
|
||||
@@ -1354,8 +1556,8 @@ class Worker(WorkerBase):
|
||||
return self.model_runner.pool() # type: ignore
|
||||
return None
|
||||
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
return self.model_runner.take_draft_token_ids()
|
||||
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||
return self.model_runner.take_draft_token_ids(step_id=step_id)
|
||||
|
||||
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
|
||||
# Check if profiling is enabled
|
||||
@@ -1580,8 +1782,16 @@ class Worker(WorkerBase):
|
||||
self.model_runner.reset_lora_state()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._ple_offload_stopping = True
|
||||
gc.unfreeze()
|
||||
|
||||
# Bounds-guard instrumentation: synchronous final read of the guard
|
||||
# counters, so a hit on the LAST step is reported before teardown.
|
||||
try:
|
||||
guard_telemetry.drain()
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("guard telemetry FINAL DRAIN failed")
|
||||
|
||||
# has_kv_transfer_group can be None during interpreter shutdown.
|
||||
if ensure_kv_transfer_shutdown is not None:
|
||||
ensure_kv_transfer_shutdown()
|
||||
@@ -1595,6 +1805,10 @@ class Worker(WorkerBase):
|
||||
|
||||
self.elastic_ep_executor.shutdown()
|
||||
|
||||
if self._ple_offload_worker_handle is not None:
|
||||
self._ple_offload_worker_handle.close()
|
||||
self._ple_offload_worker_handle = None
|
||||
|
||||
# Release GPU resources held by the model runner so that memory
|
||||
# can be reclaimed when running in-process
|
||||
if model_runner := getattr(self, "model_runner", None):
|
||||
|
||||
@@ -0,0 +1,796 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Persistent, cudagraph-safe counters for the bounds-guard instrumentation series.
|
||||
|
||||
Every guard in the series MASKS an out-of-range access so the SM cannot fault
|
||||
on it, and records that it did. A guard that masks silently turns a crash into
|
||||
a wrong answer and tells nobody, so the recording is not optional.
|
||||
|
||||
DESIGN -- follows the QSA KV-clip counter (models/qwen4_exp/nvidia/qsa.py
|
||||
:144-152, :205-225), the one instrument in this tree that already survives FULL
|
||||
cudagraph replay:
|
||||
|
||||
* counters are PERSISTENT int64 GPU tensors, allocated eagerly by prealloc()
|
||||
from the model runner constructor, so nothing is allocated inside a capture
|
||||
and the data_ptr is stable across replays;
|
||||
* the kernels do nothing but captured atomics, so replay keeps counting;
|
||||
* a daemon thread reads them on a cadence, OFF the engine thread, so no
|
||||
device->host sync is added anywhere on the step path;
|
||||
* that thread is started only AFTER warmup and cudagraph capture, from
|
||||
gpu_worker beside enable_gpu_sync_check(). See start_reader().
|
||||
|
||||
The first design allocated a fresh counter per launch and read it back
|
||||
inline with .tolist(). That was wrong twice over: the D2H aborts cudagraph
|
||||
capture at startup, and under replay the Python host code never runs, so the
|
||||
counter was simultaneously re-zeroed every step and never read. Guards 2, 3 and
|
||||
4 sit inside the captured forward (QSA declares AttentionCGSupport.UNIFORM_BATCH,
|
||||
nvidia/qsa.py:286), which is exactly where it mattered.
|
||||
|
||||
NEGATIVE VALUES. Slots 1 and 2 are written with tl.atomic_max into a zeroed
|
||||
buffer, so a negative row or index would be lost. Every recorded value is
|
||||
therefore stored biased by GUARD_BIAS and the reader subtracts it; a slot that
|
||||
is still 0 means "nothing recorded", which is distinguishable from a recorded
|
||||
0 (GUARD_BIAS). Kernel modules must bind the bias as a tl.constexpr -- a plain
|
||||
module-level int is not readable from a @triton.jit body.
|
||||
|
||||
MODES -- VLLM_E2_GUARD_MODE:
|
||||
raise the reader logs and arms a flag; a LATER execute_model on
|
||||
the engine thread raises. Mirrors VLLM_RAISE_ON_LOGIT_NANS in intent:
|
||||
a masked fault that silently returns a wrong answer is worse for a
|
||||
serving product than a loud stop. Read check_pending below before
|
||||
using it -- the raise is late and it is local to one rank.
|
||||
Targeted repro boots.
|
||||
warn (DEFAULT since v2.5) log only -- EXCEPT for the sites in
|
||||
_ALWAYS_FATAL_SITES, which arm the raise whatever the mode says,
|
||||
because their masked value has already been written somewhere durable.
|
||||
USE THIS FOR LONG STRESS BOOTS.
|
||||
off (the default in v2 through v2.2.0: the reader thread and the QSA clip
|
||||
reader cost ~0.3 ms per decode step on TP2xPP2; set
|
||||
VLLM_E2_GUARD_MODE=off to skip the reader)
|
||||
no reader thread. The kernels still mask and still count; nothing is
|
||||
read -- INCLUDING the sites in _ALWAYS_FATAL_SITES. Use only when you
|
||||
have decided you do not want the evidence.
|
||||
|
||||
WHICH RUNNER. prealloc() is called from GPUModelRunner.__init__ and
|
||||
check_pending() from its execute_model, both in vllm/v1/worker/gpu/ -- the V2
|
||||
runner. On a V1 runner the counters are allocated lazily by the first guarded
|
||||
launch instead (correct, but possibly on a capturing stream) and NOTHING calls
|
||||
check_pending, so MODE=raise degrades to MODE=warn and _ALWAYS_FATAL_SITES
|
||||
never stops anything. The guards themselves still mask and count on both. Any
|
||||
release note for this series has to say so.
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Bias applied to every recorded row/value so negatives survive atomic_max on a
|
||||
# zero-initialised buffer. Any int32-range value plus this is > 0.
|
||||
GUARD_BIAS = 1 << 31
|
||||
|
||||
# One (n_masked, row, value) triple per guarded kernel.
|
||||
SLOTS_PER_SITE = 3
|
||||
|
||||
# Largest value the bias encoding can carry without overflowing the int64 slot.
|
||||
GUARD_MAX_ENCODABLE = (1 << 63) - 1 - GUARD_BIAS
|
||||
# Smallest one. At exactly -GUARD_BIAS the encoded word is 0, which the decode
|
||||
# cannot tell from "nothing recorded"; below it the encoded word is negative
|
||||
# and tl.atomic_max against a zeroed buffer never takes it. Both cases lose the
|
||||
# VALUE and neither loses the COUNT -- see decode_slot.
|
||||
GUARD_MIN_ENCODABLE = -GUARD_BIAS + 1
|
||||
|
||||
|
||||
def encode_slot(value: int) -> int:
|
||||
"""Producer side of the bias encoding, in Python (for the limits tests).
|
||||
|
||||
The guarded kernels inline `(value + _GUARD_BIAS).to(tl.int64)` and store
|
||||
it with `tl.atomic_max`; this is the same arithmetic, exposed so the
|
||||
encoding's limits can be tested without Triton and without a GPU. It is
|
||||
NOT called by the kernels -- a Python function is not readable from a
|
||||
@triton.jit body, which is why _GUARD_BIAS is a tl.constexpr there.
|
||||
"""
|
||||
return int(value) + GUARD_BIAS
|
||||
|
||||
|
||||
def decode_slot(biased: int) -> int | None:
|
||||
"""Reader side. ``None`` means "this slot recorded nothing".
|
||||
|
||||
THE ENCODING'S LIMITS (closed here by test
|
||||
rather than by change). A slot is an int64 zeroed at allocation and only
|
||||
ever written with `tl.atomic_max`, so the recorded value must be biased to
|
||||
survive negatives, and 0 has to mean "unwritten". Three classes of extreme
|
||||
value are therefore not recoverable from the slot:
|
||||
|
||||
* ``value == -GUARD_BIAS`` (INT32_MIN) encodes to exactly 0 and is
|
||||
FALSIFIED: it reads back as "nothing recorded".
|
||||
* ``value < -GUARD_BIAS`` encodes negative, `atomic_max` against 0 keeps
|
||||
0, and the value is LOST the same way.
|
||||
* ``value > GUARD_MAX_ENCODABLE`` overflows the int64 slot.
|
||||
|
||||
In every one of those cases the COUNT slot, which carries no bias and is
|
||||
written with `atomic_add`, still advances.
|
||||
|
||||
BUT `None` IS NOT A DIAGNOSIS OF "UNENCODABLE". The reader has
|
||||
NO ordering dependency on the producer -- it copies through a pinned mirror
|
||||
on a side stream with no dependency on the compute stream -- so a count
|
||||
that has become visible with its maxima not yet visible produces exactly
|
||||
the same reading. `_poll_once`'s one-tick confirm delay makes that less
|
||||
likely; it does not make it impossible, and a delay is not a happens-before
|
||||
relationship. So `None` means UNOBSERVED OR UNENCODABLE.
|
||||
|
||||
What it never means is "this field recorded nothing while the count rose".
|
||||
Anything reading these slots must say so, and must not build a fatal
|
||||
decision on the maxima -- those decisions are count-based.
|
||||
"""
|
||||
return None if biased == 0 else int(biased) - GUARD_BIAS
|
||||
|
||||
SITE_MAMBA_COPY = 0
|
||||
SITE_GDN_DECODE_STATE = 1
|
||||
SITE_QSA_PRE_INDEXER = 2
|
||||
SITE_QSA_PREFILL = 3
|
||||
SITE_QSA_DECODE = 4
|
||||
SITE_BT_GATHER = 5
|
||||
SITE_SLOT_MAPPING = 6
|
||||
SITE_KV_ZEROER = 7
|
||||
SITE_GDN_RECURRENT_STATE = 8
|
||||
SITE_QSA_ROPE_POS = 9
|
||||
# The ring-skip telemetry: NOT a masked fault -- a suffix-commit row the writer
|
||||
# skipped because the token's state slot was < 0 or >= ring capacity.
|
||||
# The skip itself is the earlier behaviour; the count is the instrument
|
||||
# for the v2.2 ring-row reconciliation. Telemetry only: warn.
|
||||
SITE_QSA_RING_SKIP = 10
|
||||
# The pre-indexer found a historical raw-key ring row whose tag is
|
||||
# not `logical_position + 1` -- a row this request never committed for that
|
||||
# position (claim residue, a skipped suffix commit, a stale mapping). The
|
||||
# group's pooled load, its RoPE-tail read and its compressed-slot store are all
|
||||
# MASKED, so the compressed slot is left unwritten: durable damage, fatal.
|
||||
# Unlike the other sites the three slots form ONE coherent record, claimed
|
||||
# first-offender-wins with atomic_cas: slot 0 = rows flagged (all offenders),
|
||||
# slot 1 = packed (layer + 1, batch request row, source logical position) of
|
||||
# the first offender, slot 2 = the tag that offender's row actually held.
|
||||
# Decode slot 1 with decode_ring_record().
|
||||
SITE_QSA_RING_UNCOMMITTED = 11
|
||||
NUM_SITES = 12
|
||||
|
||||
# Bit layout of the SITE_QSA_RING_UNCOMMITTED record (before GUARD_BIAS).
|
||||
RING_RECORD_POS_BITS = 32
|
||||
RING_RECORD_REQ_BITS = 16
|
||||
RING_RECORD_LAYER_BITS = 14
|
||||
|
||||
|
||||
def encode_ring_record(layer_id: int, request_row: int, position: int) -> int:
|
||||
"""Python twin of the kernel's packing (tests; the kernel inlines it)."""
|
||||
layer = (int(layer_id) + 1) & ((1 << RING_RECORD_LAYER_BITS) - 1)
|
||||
row = int(request_row) & ((1 << RING_RECORD_REQ_BITS) - 1)
|
||||
pos = int(position) & ((1 << RING_RECORD_POS_BITS) - 1)
|
||||
return (
|
||||
(layer << (RING_RECORD_REQ_BITS + RING_RECORD_POS_BITS))
|
||||
| (row << RING_RECORD_POS_BITS)
|
||||
| pos
|
||||
)
|
||||
|
||||
|
||||
def decode_ring_record(biased: int) -> dict[str, int] | None:
|
||||
"""Unpack slot 1 of SITE_QSA_RING_UNCOMMITTED. ``layer`` is -1 when the
|
||||
launcher was not given one. None = nothing recorded (or not yet visible)."""
|
||||
value = decode_slot(biased)
|
||||
if value is None:
|
||||
return None
|
||||
pos_mask = (1 << RING_RECORD_POS_BITS) - 1
|
||||
req_mask = (1 << RING_RECORD_REQ_BITS) - 1
|
||||
layer_mask = (1 << RING_RECORD_LAYER_BITS) - 1
|
||||
return {
|
||||
"layer": ((value >> (RING_RECORD_REQ_BITS + RING_RECORD_POS_BITS)) & layer_mask)
|
||||
- 1,
|
||||
"request_row": (value >> RING_RECORD_POS_BITS) & req_mask,
|
||||
"source_position": value & pos_mask,
|
||||
}
|
||||
|
||||
|
||||
SITE_NAMES = {
|
||||
SITE_MAMBA_COPY: "_copy_mamba_state_block (block-table row/column)",
|
||||
SITE_GDN_DECODE_STATE: "fused_sigmoid_gating_delta_rule_update_kernel "
|
||||
"(ssm_state_indices column)",
|
||||
SITE_QSA_PRE_INDEXER: "_qsa_pre_indexer_kernel (k_work request row)",
|
||||
SITE_QSA_PREFILL: "_qsa_mqa_paged_prefill_kernel (page-table page id)",
|
||||
SITE_QSA_DECODE: "_qsa_mqa_paged_uniform_kernel (page-table page id)",
|
||||
SITE_BT_GATHER: "_gather_block_tables_kernel (request row / num_blocks)",
|
||||
SITE_SLOT_MAPPING: "_compute_slot_mappings_kernel (request-state row)",
|
||||
SITE_KV_ZEROER: "_zero_kv_blocks_kernel (KV pool block id)",
|
||||
SITE_GDN_RECURRENT_STATE: "fused_recurrent_gated_delta_rule_fwd_kernel "
|
||||
"(ssm_state_indices column)",
|
||||
SITE_QSA_ROPE_POS: "_norm_rope in _qsa_pre_indexer_kernel "
|
||||
"(rope position vs cos/sin table rows; slot 2 = axis 0=t 1=h 2=w)",
|
||||
SITE_QSA_RING_SKIP: "_qsa_pre_indexer_kernel suffix commit SKIPPED "
|
||||
"(state slot < 0 or >= ring capacity; slot 1 = token index, "
|
||||
"slot 2 = state slot value)",
|
||||
SITE_QSA_RING_UNCOMMITTED: "_qsa_pre_indexer_kernel historical ring row "
|
||||
"UNCOMMITTED (tag != logical_position + 1; group load, RoPE tail and "
|
||||
"compressed-slot store masked; slot 1 = packed first-offender record, "
|
||||
"slot 2 = the tag its row held)",
|
||||
}
|
||||
|
||||
|
||||
# Release defaults: warn, 0.05 s.
|
||||
# Earlier revisions shipped raise / 0.5 while every qualification boot set both
|
||||
# variables explicitly, so the defaults were never the thing being tested.
|
||||
# v2 through v2.2.0 shipped the code default off; v2.5 puts
|
||||
# it back to warn so the code default, this comment and the shipped launchers agree.
|
||||
_MODE = os.environ.get("VLLM_E2_GUARD_MODE", "warn").strip().lower()
|
||||
if _MODE not in ("raise", "warn", "off"):
|
||||
# fail closed: a set-but-broken gate must not silently disarm the instrument
|
||||
raise ValueError(f"VLLM_E2_GUARD_MODE must be raise|warn|off, got {_MODE!r}")
|
||||
|
||||
|
||||
def _parse_interval(text: str) -> float:
|
||||
"""Seconds between reader polls: a finite number > 0, or ValueError.
|
||||
|
||||
float() accepts "nan", "inf" and negatives; the first two reach
|
||||
time.sleep() and kill the reader daemon, a negative one raises there
|
||||
instead. Reject them at import, where a broken
|
||||
setting is a loud boot failure instead of a silently absent instrument.
|
||||
"""
|
||||
try:
|
||||
value = float(text)
|
||||
except ValueError as exc:
|
||||
raise ValueError("VLLM_E2_GUARD_INTERVAL must be seconds") from exc
|
||||
if not math.isfinite(value) or value <= 0:
|
||||
raise ValueError(
|
||||
f"VLLM_E2_GUARD_INTERVAL must be a finite number > 0, got {text!r}"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
_INTERVAL = _parse_interval(os.environ.get("VLLM_E2_GUARD_INTERVAL", "0.05"))
|
||||
|
||||
_buffers: dict[torch.device, torch.Tensor] = {}
|
||||
# Devices the reader has given up on. They stay in _buffers: see _reader.
|
||||
_read_disabled: set[torch.device] = set()
|
||||
_lock = threading.Lock()
|
||||
_reader_started = False
|
||||
_pending: str | None = None
|
||||
# Highest n_masked already reported, per (device, site). Module level, not a
|
||||
# _reader local, so reset_for_test can clear it: a test that zeroes the
|
||||
# counters but leaves this behind gets no WARNING on the next hit, because the
|
||||
# new count is below the remembered high-water mark. That is a test silently
|
||||
# measuring nothing.
|
||||
_seen: dict[tuple[torch.device, int], int] = {}
|
||||
|
||||
|
||||
class NonBlockingReader:
|
||||
"""Reads device counter tensors from a side thread without ever waiting on
|
||||
the compute stream (v2 reader fix).
|
||||
|
||||
The first reader called ``buf.tolist()`` on its cadence: a synchronous,
|
||||
pageable device-to-host copy on the compute stream. The driver services
|
||||
that copy holding the CUDA context lock until the stream's queued work has
|
||||
completed. Under the PLE pull transport the stream can hold a GPU-side
|
||||
wait whose producer needs the same lock (the connector thread's
|
||||
``event.synchronize()`` and the main thread's next kernel launch), so a
|
||||
reader tick landing inside that window deadlocked the whole worker until
|
||||
the executor timeout killed the engine (two hangs, both at
|
||||
the end of long prefills; py-spy stacks confirmed the deadlock).
|
||||
|
||||
Here every registered tensor gets a pinned host mirror and the copy runs
|
||||
on a dedicated side stream with ``non_blocking=True``; completion is
|
||||
observed with ``event.query()``. Nothing in this class blocks on the GPU.
|
||||
The side stream has no dependency on the compute stream, so a pending
|
||||
wait there cannot stall the copy, and a non-blocking pinned copy returns
|
||||
from the driver immediately. The rows a tick reports are therefore the
|
||||
values as of the previous tick's copy, and the three slots of one site
|
||||
may come from slightly different moments while kernels are still
|
||||
counting: the count is monotone and the maxima are lifetime maxima, so
|
||||
this changes nothing the reader claims.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._streams: dict[torch.device, torch.cuda.Stream] = {}
|
||||
# key -> [device_tensor, pinned_host_tensor | None, event | None, inflight]
|
||||
self._jobs: dict[object, list] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def register(self, key: object, tensor: torch.Tensor) -> None:
|
||||
with self._lock:
|
||||
if key in self._jobs:
|
||||
return
|
||||
if tensor.device.type != "cuda":
|
||||
self._jobs[key] = [tensor, None, None, False]
|
||||
return
|
||||
dev = tensor.device
|
||||
# A stream/event/pinned allocation inside a cudagraph
|
||||
# capture is not capture-safe. prealloc() registers before capture
|
||||
# and the reader thread registers lazily (it is never capturing);
|
||||
# any other caller reaching this during a capture is a bug.
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
raise RuntimeError(
|
||||
"guard_telemetry: NonBlockingReader.register() called during "
|
||||
"cudagraph capture; prealloc() must run before capture"
|
||||
)
|
||||
if dev not in self._streams:
|
||||
with torch.cuda.device(dev):
|
||||
self._streams[dev] = torch.cuda.Stream(device=dev)
|
||||
host = torch.empty(
|
||||
tensor.shape, dtype=tensor.dtype, device="cpu", pin_memory=True
|
||||
)
|
||||
with torch.cuda.device(dev):
|
||||
event = torch.cuda.Event()
|
||||
self._jobs[key] = [tensor, host, event, False]
|
||||
|
||||
def registered(self, key: object) -> bool:
|
||||
return key in self._jobs
|
||||
|
||||
def poll_one(self, key: object) -> list | None:
|
||||
"""Return the rows of a completed copy for ``key`` and start the next
|
||||
one; return None when the previous copy is still in flight. Never
|
||||
blocks."""
|
||||
job = self._jobs[key]
|
||||
tensor, host, event, inflight = job
|
||||
if host is None: # CPU tensor: nothing can block
|
||||
return tensor.tolist()
|
||||
rows = None
|
||||
if inflight:
|
||||
if not event.query():
|
||||
return None
|
||||
rows = host.tolist()
|
||||
stream = self._streams[tensor.device]
|
||||
with torch.cuda.device(tensor.device), torch.cuda.stream(stream):
|
||||
host.copy_(tensor, non_blocking=True)
|
||||
event.record(stream)
|
||||
job[3] = True
|
||||
return rows
|
||||
|
||||
def poll(self) -> dict[object, list]:
|
||||
out = {}
|
||||
with self._lock:
|
||||
keys = list(self._jobs)
|
||||
for key in keys:
|
||||
rows = self.poll_one(key)
|
||||
if rows is not None:
|
||||
out[key] = rows
|
||||
return out
|
||||
|
||||
def read_now(self, key: object) -> list:
|
||||
"""Blocking read for orderly shutdown / tests ONLY: copies through the
|
||||
pinned mirror on the side stream and waits for THAT copy (never a
|
||||
pageable copy, never a wait on the compute stream). Callers sync the
|
||||
device first when they need the latest kernel writes."""
|
||||
job = self._jobs[key]
|
||||
tensor, host, event, _ = job
|
||||
if host is None:
|
||||
return tensor.tolist()
|
||||
stream = self._streams[tensor.device]
|
||||
with torch.cuda.device(tensor.device), torch.cuda.stream(stream):
|
||||
host.copy_(tensor, non_blocking=True)
|
||||
event.record(stream)
|
||||
event.synchronize()
|
||||
job[3] = False
|
||||
return host.tolist()
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Test helper: drop in-flight copies so a zeroed counter is not
|
||||
reported from a stale mirror."""
|
||||
for job in self._jobs.values():
|
||||
if job[1] is not None:
|
||||
if job[3]:
|
||||
job[2].synchronize()
|
||||
job[1].zero_()
|
||||
job[3] = False
|
||||
|
||||
|
||||
_nb_reader = NonBlockingReader()
|
||||
# device -> the copy in which a count increase was first seen
|
||||
# (reported on the next tick)
|
||||
_confirm_pending: dict[torch.device, list] = {}
|
||||
|
||||
# Sites that arm the pending raise even under VLLM_E2_GUARD_MODE=warn. warn is
|
||||
# the right default for a guard that turns a crash into a locally wrong answer,
|
||||
# because the run continues and produces more evidence. It is the wrong default
|
||||
# for a guard whose masked value has already been WRITTEN somewhere durable --
|
||||
# the damage outlives the step and every later step reads it, so "continue and
|
||||
# collect evidence" means "collect evidence about corrupted state".
|
||||
#
|
||||
# READ THE THREE LIMITS BEFORE RELYING ON THIS. The name says
|
||||
# "always fatal"; it is not a promise of any of the following:
|
||||
#
|
||||
# 1. MODE=off NEVER ARMS IT. off starts no reader, and the reader is the only
|
||||
# thing that inspects a counter, so a site in this set is exactly as
|
||||
# unheard as any other under off. off means "the kernels mask and count and
|
||||
# nobody looks", including here. Do not set off on a boot that is relying
|
||||
# on a fatal site to stop it.
|
||||
# 2. IT IS RANK-LOCAL. Counters are per process, so only the rank whose GPU
|
||||
# fired raises. Under PP/TP its peers received the same execute_model RPC
|
||||
# and block in recv/allreduce until the NCCL watchdog fires, minutes later.
|
||||
# What you get is ONE RuntimeError on the firing rank plus a multi-rank
|
||||
# watchdog timeout, and the evidence is the WARNING in the firing rank's
|
||||
# log -- find that log first, not the watchdog traceback.
|
||||
# 3. IT IS LATE BY UP TO VLLM_E2_GUARD_INTERVAL. The reader polls; the raise
|
||||
# lands on a later step. For a durable-damage site that window is how much
|
||||
# corrupted state gets written after the first hit, so a boot with such a
|
||||
# site armed should keep VLLM_E2_GUARD_INTERVAL at 0.05 (the default), not raise
|
||||
# it. The cost is 20 host-side reads a second off the engine thread.
|
||||
_ALWAYS_FATAL_SITES: set[int] = set()
|
||||
|
||||
# A clamped rope position is not a locally wrong answer that the next step
|
||||
# forgets. On the pooled path the rotated key is STORED into the compressed
|
||||
# cache, so every later step reads a wrongly-rotated key for that block, and
|
||||
# the ring row that produced it is still there to do it again. Continuing past
|
||||
# this one collects evidence about corrupted state, so it aborts the stage
|
||||
# whatever VLLM_E2_GUARD_MODE says. (bounds-guard instrumentation, rope site)
|
||||
_ALWAYS_FATAL_SITES.add(SITE_QSA_ROPE_POS)
|
||||
|
||||
# An uncommitted ring row leaves the group's compressed slot
|
||||
# UNWRITTEN (the store is masked, there is no correct value to store), and the
|
||||
# selection kernels will read that slot on every later step. That is durable
|
||||
# damage by the same argument as the rope site, so it stops the rank under
|
||||
# warn too. The failure boundary is the rank, not the request: per-request
|
||||
# containment (discard + quarantine + TP/PP coordination) is not implemented.
|
||||
_ALWAYS_FATAL_SITES.add(SITE_QSA_RING_UNCOMMITTED)
|
||||
|
||||
|
||||
def _normalize(device: torch.device) -> torch.device:
|
||||
"""torch.device("cuda") and torch.device("cuda:0") are different dict keys.
|
||||
|
||||
A miss here would allocate a second buffer lazily, on whatever stream is
|
||||
current -- the replay defect re-entering through a typo.
|
||||
"""
|
||||
device = torch.device(device)
|
||||
if device.type == "cuda" and device.index is None:
|
||||
return torch.device("cuda", torch.cuda.current_device())
|
||||
return device
|
||||
|
||||
|
||||
def counter(site: int, device: torch.device) -> torch.Tensor:
|
||||
"""Persistent [3] int64 slice for one guarded kernel.
|
||||
|
||||
Allocation should already have happened in prealloc(); the lazy path is a
|
||||
fallback so a kernel reached from a context the runner did not construct
|
||||
still counts rather than crashes.
|
||||
"""
|
||||
device = _normalize(device)
|
||||
buf = _buffers.get(device)
|
||||
if buf is None:
|
||||
with _lock:
|
||||
buf = _buffers.get(device)
|
||||
if buf is None:
|
||||
buf = torch.zeros(
|
||||
(NUM_SITES, SLOTS_PER_SITE), dtype=torch.int64, device=device
|
||||
)
|
||||
_buffers[device] = buf
|
||||
# The reader mirrors this buffer through a pinned
|
||||
# host copy on a side stream (registered by prealloc() before
|
||||
# capture, or lazily by the reader thread -- never here, where
|
||||
# a capture may be in progress).
|
||||
return buf[site]
|
||||
|
||||
|
||||
def prealloc(device: torch.device) -> None:
|
||||
"""Allocate every site counter eagerly, before any cudagraph capture.
|
||||
|
||||
counter() allocates lazily on first use. A guarded kernel whose FIRST
|
||||
execution happens inside a capture would allocate on the capturing stream,
|
||||
which is exactly the failure mode to avoid. The model
|
||||
runner calls this at construction, so by capture time every buffer exists
|
||||
and counter() is a dict lookup plus a view.
|
||||
"""
|
||||
for site in range(NUM_SITES):
|
||||
counter(site, device)
|
||||
device = _normalize(device)
|
||||
if not _nb_reader.registered(device):
|
||||
_nb_reader.register(device, _buffers[device])
|
||||
|
||||
|
||||
# Readers that other modules own but that must obey the same rule: nothing reads a device tensor from a second
|
||||
# thread while a cudagraph capture is active. A module registers its starter here at import; start_reader() runs
|
||||
# them after capture, and one registered after that point (a late import) starts at once. Before v2.5 the QSA
|
||||
# clip-counter reader started on the first eager forward, i.e. BEFORE capture.
|
||||
_post_capture_hooks: list = []
|
||||
_post_capture_done = False
|
||||
|
||||
|
||||
def register_post_capture_hook(fn) -> None:
|
||||
"""Run fn() once after warmup and cudagraph capture (immediately if that already happened)."""
|
||||
if _post_capture_done:
|
||||
fn()
|
||||
else:
|
||||
_post_capture_hooks.append(fn)
|
||||
|
||||
|
||||
def run_post_capture_hooks() -> None:
|
||||
global _post_capture_done
|
||||
_post_capture_done = True
|
||||
hooks, _post_capture_hooks[:] = list(_post_capture_hooks), []
|
||||
for fn in hooks:
|
||||
try:
|
||||
fn()
|
||||
except Exception: # noqa: BLE001 -- one broken starter must not stop the others or the engine
|
||||
logger.exception("post-capture reader start failed: %r", fn)
|
||||
|
||||
|
||||
def start_reader() -> None:
|
||||
"""Start the cadence reader. Called AFTER warmup and cudagraph capture.
|
||||
|
||||
Also runs the post-capture hooks of the other readers (register_post_capture_hook), whatever the guard mode.
|
||||
|
||||
The reader must not exist during capture. It copies device counters to the
|
||||
host, and a device-to-host copy from a second thread while a capture is
|
||||
active is exactly the call class cudaStreamCaptureModeGlobal -- the default
|
||||
for torch.cuda.graph -- rejects. The QSA clip-counter reader survives it in
|
||||
practice, but that one is opt-in and this one is on by default, so
|
||||
the precedent is evidence, not proof. Starting after capture deletes the
|
||||
question rather than arguing it.
|
||||
|
||||
Between prealloc() and this call the kernels still count; nothing reads.
|
||||
"""
|
||||
global _reader_started
|
||||
run_post_capture_hooks()
|
||||
if _reader_started or _MODE == "off":
|
||||
return
|
||||
_reader_started = True
|
||||
threading.Thread(target=_reader, name="bounds-guard-reader", daemon=True).start()
|
||||
logger.info(
|
||||
"bounds-guard reader started (mode=%s, interval=%.3gs).",
|
||||
_MODE,
|
||||
_INTERVAL,
|
||||
)
|
||||
|
||||
|
||||
def _reader() -> None:
|
||||
"""Off-engine-thread cadence reader; drain() is the only other counter read."""
|
||||
while True:
|
||||
try:
|
||||
time.sleep(_INTERVAL)
|
||||
_poll_once()
|
||||
except Exception: # noqa: BLE001 -- the daemon itself must survive
|
||||
# A reader that dies is an instrument
|
||||
# that is not there, and _ALWAYS_FATAL_SITES depends on it. Arm
|
||||
# the stop so the engine thread learns that the fatal-site
|
||||
# contract is no longer held, then keep polling.
|
||||
logger.exception("bounds-guard reader loop failed")
|
||||
_arm(
|
||||
"bounds-guard reader loop failed; the fatal-site contract "
|
||||
"(_ALWAYS_FATAL_SITES) can no longer be enforced."
|
||||
)
|
||||
time.sleep(max(_INTERVAL, 0.05))
|
||||
|
||||
|
||||
def _report_rows(device: torch.device, rows: list[list[int]]) -> None:
|
||||
"""Report every (device, site) whose masked count advanced. Shared by the
|
||||
cadence reader and the final drain."""
|
||||
for site, (n_masked, biased_row, biased_val) in enumerate(rows):
|
||||
if n_masked <= _seen.get((device, site), 0):
|
||||
continue
|
||||
_seen[(device, site)] = n_masked
|
||||
row = decode_slot(biased_row)
|
||||
val = decode_slot(biased_val)
|
||||
if site == SITE_QSA_RING_UNCOMMITTED:
|
||||
# One coherent first-offender record, not independent maxima.
|
||||
record = decode_ring_record(biased_row)
|
||||
msg = (
|
||||
f"ring-row tag guard masked {n_masked} uncommitted historical "
|
||||
f"ring row(s) in {SITE_NAMES[site]} on {device}; first "
|
||||
f"offender {record}, its row held tag {val} (expected "
|
||||
f"source_position + 1). The group was not pooled and its "
|
||||
f"compressed slot was NOT written."
|
||||
)
|
||||
logger.warning("%s", msg)
|
||||
_arm(msg)
|
||||
continue
|
||||
msg = (
|
||||
f"bounds guard masked {n_masked} out-of-range access(es) "
|
||||
f"in {SITE_NAMES.get(site, site)} on {device} "
|
||||
f"(cumulative maxima: row-field {row}, index-field {val}; these "
|
||||
f"are independent lifetime maxima, not one observed pair). The "
|
||||
f"access was suppressed, so this is a MASKED FAULT and not a "
|
||||
f"correct result -- the metadata is wrong upstream."
|
||||
)
|
||||
if row is None or val is None:
|
||||
# Corrected wording. A None field has TWO
|
||||
# possible causes and this reader cannot distinguish them, because
|
||||
# it has no happens-before relationship with the producer:
|
||||
# * UNOBSERVED -- the kernel incremented the count and its
|
||||
# atomic_max on this slot is not yet visible to this copy;
|
||||
# * UNENCODABLE -- the value was == INT32_MIN, below it, or past
|
||||
# GUARD_MAX_ENCODABLE, so the bias encoding could not carry it.
|
||||
# Either way it does NOT mean the field recorded nothing. The
|
||||
# earlier wording claimed "unencodable" outright, which the
|
||||
# one-tick confirm delay does not establish.
|
||||
msg += (
|
||||
" A None field is UNOBSERVED OR UNENCODABLE, not absent: "
|
||||
"either this copy predates the slot's atomic_max becoming "
|
||||
"visible, or the value was == INT32_MIN, below it, or past "
|
||||
f"{GUARD_MAX_ENCODABLE}. The reader has no ordering "
|
||||
"dependency on the producer, so it cannot tell which; see "
|
||||
"decode_slot."
|
||||
)
|
||||
logger.warning("%s", msg)
|
||||
if _MODE == "raise" or site in _ALWAYS_FATAL_SITES:
|
||||
_arm(msg)
|
||||
|
||||
|
||||
def _poll_once() -> None:
|
||||
for device, buf in list(_buffers.items()):
|
||||
if device in _read_disabled:
|
||||
continue
|
||||
try:
|
||||
# A non-blocking side-stream copy; None means the
|
||||
# previous copy is still in flight (report on a later tick).
|
||||
if not _nb_reader.registered(device):
|
||||
_nb_reader.register(device, buf) # reader thread: never capturing
|
||||
rows = _nb_reader.poll_one(device)
|
||||
if rows is None:
|
||||
continue
|
||||
# Caveat: the side-stream copy is not a coherent snapshot of a
|
||||
# site's three slots -- the kernels write the count first, then
|
||||
# the maxima. A copy taken between them pairs a new count with
|
||||
# stale maxima. So a count increase seen in copy k is REPORTED
|
||||
# from copy k+1 (one tick later, >= 50 ms after the increment,
|
||||
# by which time the maxima of that hit are written). The fatal
|
||||
# decision is count-based either way; only its report waits a tick.
|
||||
pend = _confirm_pending.get(device)
|
||||
rising = [
|
||||
site
|
||||
for site, (n, _r, _v) in enumerate(rows)
|
||||
if n > _seen.get((device, site), 0)
|
||||
]
|
||||
if pend is None:
|
||||
if rising:
|
||||
_confirm_pending[device] = rows
|
||||
continue
|
||||
_confirm_pending.pop(device, None)
|
||||
# report with the CONFIRMING copy's values (never lower than the
|
||||
# pending copy's counts: the counters are monotone)
|
||||
rows = [[max(a[0], b[0]), b[1], b[2]] for a, b in zip(pend, rows)]
|
||||
except Exception: # noqa: BLE001 -- a reader must never kill the run
|
||||
# Never silent: a reader that cannot read is an instrument that
|
||||
# is not there, and this whole series depends on it. Log once,
|
||||
# then stop polling that device.
|
||||
#
|
||||
# STOP POLLING IT, NOT STOP HOLDING IT. A prior version
|
||||
# popped the buffer here, which drops the only durable Python
|
||||
# reference: counter() hands out transient buf[site] views that
|
||||
# are released after each launch, and under FULL replay nothing
|
||||
# holds one at all. The storage would return to the caching
|
||||
# allocator, be reused, and the CAPTURED atomics would keep
|
||||
# writing to the baked address -- the memory-corrupting half of
|
||||
# replay defect, re-entering through the error handler.
|
||||
logger.exception(
|
||||
"bounds-guard reader failed to read the counter on %s; "
|
||||
"no further guard telemetry will be reported for it. The "
|
||||
"buffer is kept alive: captured graphs still write to it.",
|
||||
device,
|
||||
)
|
||||
_read_disabled.add(device)
|
||||
# Losing the reader on a device
|
||||
# silently voids the fatal-site contract for it. Arm the stop
|
||||
# regardless of MODE (off never reaches here: no reader).
|
||||
_arm(
|
||||
f"bounds-guard reader lost the counter on {device}; the "
|
||||
f"fatal-site contract (_ALWAYS_FATAL_SITES) is no longer "
|
||||
f"enforced there."
|
||||
)
|
||||
continue
|
||||
_report_rows(device, rows)
|
||||
|
||||
|
||||
def _arm(msg: str) -> None:
|
||||
global _pending
|
||||
if _pending is None:
|
||||
_pending = msg
|
||||
|
||||
|
||||
def check_pending() -> None:
|
||||
"""Engine-thread check. One Python bool per step; adds no sync.
|
||||
|
||||
Called at the top of execute_model, ahead of anything that mutates state
|
||||
and ahead of any collective, so the step it lands on aborts cleanly.
|
||||
|
||||
TWO THINGS IT DOES NOT DO -- an earlier docstring claimed one of them and
|
||||
was wrong:
|
||||
|
||||
1. It does NOT raise on the step that fired. The reader polls on a cadence,
|
||||
so the raise lands up to VLLM_E2_GUARD_INTERVAL later, hundreds of
|
||||
decode steps on. The scheduler state at the raise belongs to some later
|
||||
step, not the offending one. Attribution comes from the counter and the
|
||||
WARNING; the raise is a stop, not a snapshot.
|
||||
2. It does NOT coordinate across ranks. The counters are per process, so
|
||||
only the rank whose GPU fired arms. Its PP/TP peers received the same
|
||||
execute_model RPC, proceed into the forward, and block in recv/allreduce
|
||||
until the NCCL watchdog fires -- minutes later, as a multi-rank timeout
|
||||
that is a worse diagnostic than the Xid being chased.
|
||||
|
||||
Hence: VLLM_E2_GUARD_MODE=warn for long stress boots; raise only for a
|
||||
targeted repro where losing the group is acceptable and stopping is the
|
||||
point.
|
||||
"""
|
||||
global _pending
|
||||
if _pending is None:
|
||||
return
|
||||
msg = _pending
|
||||
_pending = None
|
||||
raise RuntimeError(
|
||||
f"{msg} (Sites in _ALWAYS_FATAL_SITES and reader failures stop the "
|
||||
f"rank under every mode except off; MODE=warn only softens the other "
|
||||
f"sites.)"
|
||||
)
|
||||
|
||||
|
||||
def drain(*, raise_on_fatal: bool = False) -> dict[str, list[int]]:
|
||||
"""Synchronous final read of every counter, off the cadence (added
|
||||
later).
|
||||
|
||||
The cadence reader can miss a hit on the LAST step of a run: the forward
|
||||
completes, its result is consumed, no further execute_model polls
|
||||
check_pending, and the process exits before the reader's next tick. Call
|
||||
this at stage end / orderly shutdown, outside capture. It synchronizes the
|
||||
device, reports anything the reader has not, and returns the unbiased
|
||||
snapshot. With raise_on_fatal it raises for a hit at a site in
|
||||
_ALWAYS_FATAL_SITES or, under MODE=raise, at any site. MODE=off drains
|
||||
nothing (no reporting under off, by contract).
|
||||
"""
|
||||
if _MODE == "off":
|
||||
return {}
|
||||
fatal: list[str] = []
|
||||
for device, buf in list(_buffers.items()):
|
||||
try:
|
||||
if device.type == "cuda":
|
||||
torch.cuda.synchronize(device)
|
||||
if not _nb_reader.registered(device):
|
||||
_nb_reader.register(device, buf)
|
||||
rows = _nb_reader.read_now(device) # pinned mirror, side stream only
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("bounds-guard FINAL DRAIN could not read %s", device)
|
||||
fatal.append(f"reader lost {device}")
|
||||
continue
|
||||
_report_rows(device, rows)
|
||||
for site, (n_masked, _r, _v) in enumerate(rows):
|
||||
if n_masked and (_MODE == "raise" or site in _ALWAYS_FATAL_SITES):
|
||||
fatal.append(f"{SITE_NAMES.get(site, site)} on {device}: {n_masked}")
|
||||
snap = snapshot()
|
||||
if fatal:
|
||||
logger.error(
|
||||
"bounds-guard FINAL DRAIN: fatal-site hits %s (snapshot %s)",
|
||||
fatal,
|
||||
snap,
|
||||
)
|
||||
if raise_on_fatal:
|
||||
raise RuntimeError(f"bounds-guard FINAL DRAIN: fatal-site hits {fatal}")
|
||||
else:
|
||||
logger.info("bounds-guard FINAL DRAIN: no fatal-site hits (snapshot %s)", snap)
|
||||
return snap
|
||||
|
||||
|
||||
def snapshot() -> dict[str, list[int]]:
|
||||
"""Test/diagnostic helper: current counters, unbiased. Syncs -- not for the
|
||||
step path."""
|
||||
out: dict[str, list[int]] = {}
|
||||
for device, buf in _buffers.items():
|
||||
for site, (n, row, val) in enumerate(buf.tolist()):
|
||||
if n:
|
||||
out[f"{device}:{SITE_NAMES.get(site, site)}"] = [
|
||||
n,
|
||||
row - GUARD_BIAS if row else 0,
|
||||
val - GUARD_BIAS if val else 0,
|
||||
]
|
||||
return out
|
||||
|
||||
|
||||
def reset_for_test() -> None:
|
||||
"""Zero every counter, clear the reader's high-water marks, and disarm.
|
||||
|
||||
All three, or the next hit is invisible: the reader only reports a count
|
||||
ABOVE what it last saw.
|
||||
"""
|
||||
global _pending
|
||||
for buf in _buffers.values():
|
||||
buf.zero_()
|
||||
_nb_reader.reset()
|
||||
_confirm_pending.clear()
|
||||
_seen.clear()
|
||||
_pending = None
|
||||
+687
-10
@@ -7,6 +7,7 @@ from typing import Any, NamedTuple
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
@@ -26,14 +27,299 @@ from vllm.v1.kv_cache_interface import (
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.utils import CpuGpuBuffer
|
||||
from vllm.v1.worker import guard_telemetry
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Bias so a negative row/index survives atomic_max on a zeroed counter
|
||||
# -- it MUST be a tl.constexpr: a plain module-level int is not readable
|
||||
# from a @triton.jit body -- an earlier build shipped one and every guarded kernel died
|
||||
# with "NameError: Cannot access global variable _GUARD_BIAS" on the smoke.
|
||||
_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS)
|
||||
|
||||
# Production value for the align post-process's block-table row rule. True
|
||||
# means "the row is the request's own state slot", which is the only correct
|
||||
# rule for the PP-deferred launch (see run_fused_postprocess_align). It is a
|
||||
# module-level name rather than a literal at the launch site for exactly one
|
||||
# reason: the failing control for the slot-ownership check below has to be able
|
||||
# to flip it. NOTHING reads it from the environment, so there is no way to turn
|
||||
# it off on a serving host; tests monkeypatch it.
|
||||
ALIGN_BT_ROW_IS_REQ = True
|
||||
|
||||
# Host-side mamba state-slot ownership check (PLE diagnostics, llama.cpp#29092 class /
|
||||
# vllm#55506 failure mode). off | warn | raise, default off, so it costs
|
||||
# nothing in production. It is a DIAGNOSTIC, not a device guard: it copies
|
||||
# decision arrays and block-table rows to the host, which synchronizes.
|
||||
#
|
||||
# WHY IT EXISTS. Mamba state blocks are the one KV class with no zeroing and no
|
||||
# ownership instrument:
|
||||
# * single_type_kv_cache_manager.py:96-99 records newly allocated blocks for
|
||||
# zeroing only when the spec `isinstance(kv_cache_spec, AttentionSpec)`.
|
||||
# `MambaSpec` is not an `AttentionSpec`, so mamba state blocks are NEVER
|
||||
# zeroed by `KVBlockZeroer` (gpu/model_runner.py:1213).
|
||||
# * the host-side KV-zeroer ownership check (sched/scheduler.py:1508-1535) only
|
||||
# inspects `new_block_ids_to_zero`, a list mamba blocks are never on.
|
||||
# * a genuinely fresh request is still safe: gdn_attn.py:401 sets
|
||||
# `has_initial_state = context_lens_tensor > 0`, so at context_len 0 the
|
||||
# GDN kernel ignores whatever bytes the reused block holds.
|
||||
# * a RESUMING request is not. With `has_initial_state` true the kernel
|
||||
# consumes the block's contents, and the only thing that puts the right
|
||||
# contents there is this align copy. Everything else we have bounds
|
||||
# indices (the bounds guards): a bounds check catches an
|
||||
# out-of-range row, never a wrong-but-in-range one, which is the whole of
|
||||
# this defect class.
|
||||
# Read through the envs registry, not os.getenv: the registry owns the
|
||||
# normalization, so the value reported by
|
||||
# `vllm.envs` and the value this check applies cannot drift apart. It is
|
||||
# registered as a non-compile factor there, and an unrecognised value raises
|
||||
# on first read rather than silently disarming the instrument.
|
||||
_SLOT_OWNERSHIP_MODE: str = envs.VLLM_E2_MAMBA_SLOT_OWNERSHIP
|
||||
# Liveness cadence: first check and every Nth are logged with a running count
|
||||
# (see MambaSpecDecodeGPUContext.slot_ownership_checks_run).
|
||||
_E2_CHECK_HEARTBEAT = 4096
|
||||
|
||||
_NULL_BLOCK_ID = 0
|
||||
|
||||
|
||||
def mamba_slot_ownership_violations(
|
||||
*,
|
||||
num_reqs: int,
|
||||
idx_mapping: list[int],
|
||||
bt_row_is_req: bool,
|
||||
block_tables: list[list[list[int]]],
|
||||
num_rows: int,
|
||||
num_accepted_tokens: list[int],
|
||||
state_idx: list[int],
|
||||
new_num_computed_tokens: list[int],
|
||||
block_size: int,
|
||||
registered_ptrs: list[int] | None = None,
|
||||
view_ptrs: list[int] | None = None,
|
||||
) -> list[str]:
|
||||
"""ALIGN ROW-RULE AND SIMULTANEOUS BLOCK-COLLISION CHECK.
|
||||
|
||||
Read the name literally. It was narrowed from "slot ownership":
|
||||
with the production row rule ``row_used == req_idx``, so the
|
||||
destination comparison below reduces to ``table[req_idx][dst_col]``
|
||||
against itself and is TAUTOLOGICAL on a healthy run. This function derives
|
||||
what it expects and what it observes from THE SAME TABLE, and a table
|
||||
cannot testify about its own staleness. What it does and does not
|
||||
establish, stated so that zero findings are not over-read:
|
||||
|
||||
ESTABLISHES
|
||||
|
||||
1. **Row attribution.** The row the kernel will use is the request's own
|
||||
state slot. This is the property vllm#55506 violates: under PP the
|
||||
deferred launch carries an ``idx_mapping`` captured ``pp_size`` steps
|
||||
earlier, so ``batch_idx`` names a different request. The
|
||||
``BT_ROW_IS_REQ=False`` control exercises exactly this and nothing
|
||||
wider.
|
||||
2. **Simultaneous block collision.** Within ONE step, no two live request
|
||||
slots reach the same state block -- counting both the source and the
|
||||
destination column, and counting slots that are live but not copying,
|
||||
because a writer colliding with a non-copying live owner is the same
|
||||
corruption. ``NULL_BLOCK_ID`` is excluded: block id 0 is the pool's
|
||||
placeholder (``core/block_pool.py:188-191``), owned by no request.
|
||||
3. **Pointer-registration agreement**, when the caller supplies
|
||||
``registered_ptrs``: the tables read here are the tables whose addresses
|
||||
the kernel was handed. That is the one genuinely INDEPENDENT source of
|
||||
truth available at this layer, and without it a wrong pointer array
|
||||
would be invisible -- the checker would faithfully validate a table the
|
||||
kernel never reads.
|
||||
|
||||
DOES NOT ESTABLISH -- do not quote zero findings as covering these:
|
||||
|
||||
* **Stale ownership across steps.** The block-table row could itself be
|
||||
stale; nothing here compares it against the scheduler's allocation.
|
||||
* **Request-generation reuse.** A slot re-issued to a different request
|
||||
with the same index is indistinguishable here: there is no generation
|
||||
id in any tensor this function sees.
|
||||
* **Cross-step source staleness.** A source block correct for this step but
|
||||
written by a previous request's state is invisible.
|
||||
* **Anything about a captured replay**, which does not re-enter the caller.
|
||||
|
||||
Closing those needs an independent allocation/generation map from the
|
||||
scheduler side, which is a plumbing change, not a diagnostic.
|
||||
|
||||
Pure and side-effect free; every argument is a plain Python list or int, so
|
||||
this runs with no GPU and no CUDA context. The caller owns the device->host
|
||||
copies and the mode.
|
||||
|
||||
``block_tables[g][row][col]`` is the PERSISTENT per-req-slot table of mamba
|
||||
group ``g``. ``registered_ptrs`` and ``view_ptrs``, when given, are the
|
||||
kernel's ``persistent_block_table_ptrs`` values and the ``data_ptr()`` of
|
||||
the tables actually read, compared elementwise.
|
||||
|
||||
Returns one string per violation, empty when clean.
|
||||
"""
|
||||
violations: list[str] = []
|
||||
# (group, block id) -> (req slot, which column class claimed it). Both the
|
||||
# source and the destination column are entered, and live-but-not-copying
|
||||
# slots are entered too, so a writer colliding with a non-copying owner is
|
||||
# caught (the destination-only map missed that case).
|
||||
block_owner: dict[tuple[int, int], tuple[int, str]] = {}
|
||||
|
||||
# (3) pointer-registration agreement -- the one independent source of truth
|
||||
# available here. Without it the checker can faithfully validate a table
|
||||
# the kernel never reads.
|
||||
if registered_ptrs is not None and view_ptrs is not None:
|
||||
if len(registered_ptrs) != len(view_ptrs):
|
||||
violations.append(
|
||||
f"pointer registration: {len(registered_ptrs)} registered "
|
||||
f"block-table pointer(s) but {len(view_ptrs)} retained view(s)"
|
||||
)
|
||||
for group_idx, (registered, view) in enumerate(
|
||||
zip(registered_ptrs, view_ptrs)
|
||||
):
|
||||
if int(registered) != int(view):
|
||||
violations.append(
|
||||
f"pointer registration: group {group_idx} kernel reads "
|
||||
f"0x{int(registered):x} but the table checked here is at "
|
||||
f"0x{int(view):x}. Every other finding in this run is "
|
||||
f"about the wrong tensor."
|
||||
)
|
||||
|
||||
def _claim(group_idx: int, block: int, req_idx: int, kind: str) -> None:
|
||||
"""Record one live claim on a state block and report a collision."""
|
||||
if block == _NULL_BLOCK_ID:
|
||||
return # the pool's placeholder; owned by no request
|
||||
key = (group_idx, block)
|
||||
previous = block_owner.get(key)
|
||||
if previous is not None and previous[0] != req_idx:
|
||||
violations.append(
|
||||
f"block collision: group {group_idx}, state block {block} is "
|
||||
f"claimed in one step by request slot {previous[0]} "
|
||||
f"({previous[1]}) and request slot {req_idx} ({kind}) "
|
||||
f"(llama.cpp#29092 class: a prior request's recurrent state in "
|
||||
f"a slot another request resumes into)."
|
||||
)
|
||||
elif previous is None:
|
||||
block_owner[key] = (req_idx, kind)
|
||||
|
||||
def _cell(group_idx: int, table, row: int, col: int, what: str) -> int | None:
|
||||
"""Bounded read. Returns None and reports rather than indexing wrongly.
|
||||
|
||||
A NEGATIVE column silently indexes a Python list from the end, so an
|
||||
unbounded read here invents a block id and can report an unrelated
|
||||
collision (example: accepted 1 with computed 0 yields dst_col -1,
|
||||
which the device guard masks and this reader would turn into the LAST
|
||||
column). Both bounds, both indices, every time.
|
||||
"""
|
||||
if not 0 <= row < len(table):
|
||||
violations.append(
|
||||
f"malformed metadata: group {group_idx}, {what} row {row} is "
|
||||
f"outside the table's [0, {len(table)}) rows"
|
||||
)
|
||||
return None
|
||||
if not 0 <= col < len(table[row]):
|
||||
violations.append(
|
||||
f"malformed metadata: group {group_idx}, {what} column {col} "
|
||||
f"is outside row {row}'s [0, {len(table[row])}) columns. The "
|
||||
f"device guard masks this access; it is reported, not read."
|
||||
)
|
||||
return None
|
||||
return int(table[row][col])
|
||||
|
||||
# Pass 1: every LIVE slot's current state block, whether or not it copies.
|
||||
# A slot that does not copy this step still owns its block, and a copying
|
||||
# writer that lands on it is the corruption -- so the claims have to be
|
||||
# entered before any writer is judged.
|
||||
for batch_idx in range(min(num_reqs, len(idx_mapping))):
|
||||
slot = int(idx_mapping[batch_idx])
|
||||
if slot < 0 or slot >= len(state_idx):
|
||||
continue
|
||||
for group_idx, table in enumerate(block_tables):
|
||||
block = _cell(group_idx, table, slot, int(state_idx[slot]), "live src")
|
||||
if block is not None:
|
||||
_claim(group_idx, block, slot, "live source")
|
||||
|
||||
for batch_idx in range(num_reqs):
|
||||
if batch_idx >= len(idx_mapping):
|
||||
violations.append(
|
||||
f"idx_mapping is shorter ({len(idx_mapping)}) than num_reqs "
|
||||
f"({num_reqs}); batch row {batch_idx} has no request slot"
|
||||
)
|
||||
continue
|
||||
req_idx = int(idx_mapping[batch_idx])
|
||||
if req_idx < 0:
|
||||
continue # the kernel's own inactive-row sentinel
|
||||
if req_idx >= min(
|
||||
len(num_accepted_tokens), len(state_idx), len(new_num_computed_tokens)
|
||||
):
|
||||
violations.append(
|
||||
f"slot bound: batch row {batch_idx} maps to request slot "
|
||||
f"{req_idx}, past the per-slot decision arrays "
|
||||
f"({len(num_accepted_tokens)}, {len(state_idx)}, "
|
||||
f"{len(new_num_computed_tokens)} entries)"
|
||||
)
|
||||
continue
|
||||
|
||||
# Mirror of the kernel's decision block (postprocess_mamba_fused_kernel,
|
||||
# the PRECOMPUTED_NEW_COMPUTED branch). Every array is read at req_idx,
|
||||
# never at batch_idx -- that asymmetry is the bug's hiding place.
|
||||
num_accepted = int(num_accepted_tokens[req_idx])
|
||||
src_col = int(state_idx[req_idx])
|
||||
new_num_computed = int(new_num_computed_tokens[req_idx])
|
||||
num_tokens_running_state = new_num_computed - num_accepted + 1
|
||||
aligned_new_computed = (new_num_computed // block_size) * block_size
|
||||
if aligned_new_computed < num_tokens_running_state:
|
||||
continue # needs_copy is False: the block table is never read
|
||||
accept_token_bias = aligned_new_computed - num_tokens_running_state
|
||||
dst_col = aligned_new_computed // block_size - 1
|
||||
if src_col == dst_col and accept_token_bias == 0:
|
||||
continue # no-op self-copy: the kernel returns before the copy
|
||||
|
||||
row_used = req_idx if bt_row_is_req else batch_idx
|
||||
|
||||
# (1) row attribution
|
||||
if row_used != req_idx:
|
||||
violations.append(
|
||||
f"row attribution: batch row {batch_idx} is request slot "
|
||||
f"{req_idx}, but the align post-process will read block-table "
|
||||
f"row {row_used}. Under PP the idx_mapping was captured "
|
||||
f"pp_size steps before this launch, so row {row_used} belongs "
|
||||
f"to a different micro-batch's request (vllm#55506)."
|
||||
)
|
||||
if not 0 <= row_used < num_rows:
|
||||
violations.append(
|
||||
f"row bound: batch row {batch_idx} resolves to block-table row "
|
||||
f"{row_used}, outside [0, {num_rows})"
|
||||
)
|
||||
continue
|
||||
|
||||
for group_idx, table in enumerate(block_tables):
|
||||
owner_block = _cell(group_idx, table, req_idx, dst_col, "owner dst")
|
||||
used_block = _cell(group_idx, table, row_used, dst_col, "used dst")
|
||||
if owner_block is None or used_block is None:
|
||||
continue
|
||||
# (2) destination agreement. NOTE: with the production row rule
|
||||
# row_used == req_idx, so this comparison is tautological on a
|
||||
# healthy run and proves nothing on its own. It earns
|
||||
# its place only in the wrong-row world, where it reports the
|
||||
# defect in BLOCK IDS rather than in indices -- which is what says
|
||||
# the copy really would land on another request's state.
|
||||
if used_block != owner_block:
|
||||
violations.append(
|
||||
f"destination disagreement: group {group_idx}, request "
|
||||
f"slot {req_idx} would resume into state block "
|
||||
f"{used_block} (row {row_used}, col {dst_col}) but its own "
|
||||
f"row holds block {owner_block} there. The copy would "
|
||||
f"write this request's recurrent state over another "
|
||||
f"request's block."
|
||||
)
|
||||
_claim(group_idx, used_block, req_idx, f"align destination col {dst_col}")
|
||||
|
||||
return violations
|
||||
|
||||
# 16 saturates HBM on H100/GB200 across the reqs=8..128 range in
|
||||
# microbenchmarks
|
||||
_TEMPORAL_TILES = 16
|
||||
|
||||
# Row bound used when no trustworthy row count is available: large enough never
|
||||
# to reject a real request slot, so the guard degrades to "reject negatives"
|
||||
# rather than to "reject correct metadata". See initialize_from_forward_context.
|
||||
_UNBOUNDED_ROWS = 1 << 30
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_requests"])
|
||||
def get_aligned_state_indices_multi_group_kernel(
|
||||
@@ -187,6 +473,11 @@ def _reinterpret_u64_as_i64(value: int) -> int:
|
||||
return value if value < (1 << 63) else value - (1 << 64)
|
||||
|
||||
|
||||
def _reinterpret_i64_as_u64(value: int) -> int:
|
||||
"""Inverse of ``_reinterpret_u64_as_i64``, for comparing pointers host side."""
|
||||
return value + (1 << 64) if value < 0 else value
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _copy_mamba_state_block(
|
||||
state_idx,
|
||||
@@ -196,6 +487,8 @@ def _copy_mamba_state_block(
|
||||
token_bias,
|
||||
block_table_ptrs_ptr,
|
||||
block_table_stride_req,
|
||||
block_table_num_rows,
|
||||
guard_ctr,
|
||||
state_base_addrs_ptr,
|
||||
state_block_strides_ptr,
|
||||
state_elem_sizes_ptr,
|
||||
@@ -243,12 +536,39 @@ def _copy_mamba_state_block(
|
||||
group_idx = tl.load(state_group_indices_ptr + state_idx).to(tl.int64)
|
||||
group_base_addr = tl.load(block_table_ptrs_ptr + group_idx)
|
||||
block_table_typed = group_base_addr.to(tl.pointer_type(tl.int32))
|
||||
# Bounds-guard instrumentation: bound the ROW as well as the column. #50021 bounds which
|
||||
# column is read within a row; this bounds which row. CAN fire on
|
||||
# postprocess_mamba_fused_kernel, where bt_row_idx comes from a device
|
||||
# idx_mapping value. CANNOT fire on precopy_mamba_align_fused_kernel, whose
|
||||
# bt_row_idx is program_id(0) bounded by num_reqs <= rows -- that path has a
|
||||
# TEMPORAL staleness problem, fixed separately, not a spatial one.
|
||||
if (bt_row_idx < 0) or (bt_row_idx >= block_table_num_rows):
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (bt_row_idx).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, _GUARD_BIAS + (-1))
|
||||
return
|
||||
block_table_base = block_table_typed + bt_row_idx * block_table_stride_req
|
||||
|
||||
# Widen block ids to int64 before they reach `block_id * state_block_stride`
|
||||
# below: state_block_stride can exceed 2**31 bytes for large mamba caches,
|
||||
# and Triton would otherwise do the multiply in int32 and wrap.
|
||||
dest_block_id = tl.load(block_table_base + dst_col).to(tl.int64)
|
||||
# PR #50021 (upstream :178): these columns are derived from per-request
|
||||
# accepted-token counts. A wrong count walks the read past this request's
|
||||
# row; the loaded int32 is then multiplied by state_block_stride and
|
||||
# dereferenced, so an out-of-range column becomes a wild (and possibly
|
||||
# misaligned) address rather than merely a wrong copy. Only the
|
||||
# out-of-range case is counted: `<= 0` also matches NULL_BLOCK_ID (0),
|
||||
# which is the ordinary unallocated marker and not a fault.
|
||||
dst_col_ok = (dst_col >= 0) & (dst_col < block_table_stride_req)
|
||||
if (dst_col < 0) or (dst_col >= block_table_stride_req):
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (bt_row_idx).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, (dst_col).to(tl.int64) + _GUARD_BIAS)
|
||||
dest_block_id = tl.load(
|
||||
block_table_base + dst_col, mask=dst_col_ok, other=-1
|
||||
).to(tl.int64)
|
||||
if dest_block_id <= 0:
|
||||
return
|
||||
dst_addr = state_base_addr + dest_block_id * state_block_stride
|
||||
|
||||
is_conv_state = conv_width > 0
|
||||
@@ -259,7 +579,16 @@ def _copy_mamba_state_block(
|
||||
if tile_idx > 0:
|
||||
return
|
||||
# DS conv layout: state_len is the slide axis; copy per dim row.
|
||||
src_block_id = tl.load(block_table_base + src_col).to(tl.int64)
|
||||
src_col_ok = (src_col >= 0) & (src_col < block_table_stride_req)
|
||||
if (src_col < 0) or (src_col >= block_table_stride_req):
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (bt_row_idx).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, (src_col).to(tl.int64) + _GUARD_BIAS)
|
||||
src_block_id = tl.load(
|
||||
block_table_base + src_col, mask=src_col_ok, other=-1
|
||||
).to(tl.int64)
|
||||
if src_block_id <= 0:
|
||||
return
|
||||
dim_rows = tl.load(state_dim_row_count_ptr + state_idx)
|
||||
row_stride = tl.load(state_dim_row_stride_ptr + state_idx)
|
||||
src_block_addr = state_base_addr + src_block_id * state_block_stride
|
||||
@@ -310,7 +639,16 @@ def _copy_mamba_state_block(
|
||||
# SD conv: copy
|
||||
# state[bt[src_col], token_bias:] ->
|
||||
# state[bt[dst_col], :conv_width - token_bias]
|
||||
src_block_id = tl.load(block_table_base + src_col).to(tl.int64)
|
||||
src_col_ok = (src_col >= 0) & (src_col < block_table_stride_req)
|
||||
if (src_col < 0) or (src_col >= block_table_stride_req):
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (bt_row_idx).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, (src_col).to(tl.int64) + _GUARD_BIAS)
|
||||
src_block_id = tl.load(
|
||||
block_table_base + src_col, mask=src_col_ok, other=-1
|
||||
).to(tl.int64)
|
||||
if src_block_id <= 0:
|
||||
return
|
||||
src_block_addr = state_base_addr + src_block_id * state_block_stride
|
||||
token_bytes = state_inner_size * state_elem_size
|
||||
num_dst_tokens = conv_width - token_bias
|
||||
@@ -349,7 +687,17 @@ def _copy_mamba_state_block(
|
||||
# Temporal state: copy state[bt[src_col + token_bias]] -> state[bt[dst_col]]
|
||||
# Body u64 range is partitioned across TEMPORAL_TILES CTAs to keep the
|
||||
# SMs filled at small batch.
|
||||
actual_src_block_id = tl.load(block_table_base + src_col + token_bias).to(tl.int64)
|
||||
tmp_col = src_col + token_bias
|
||||
tmp_col_ok = (tmp_col >= 0) & (tmp_col < block_table_stride_req)
|
||||
if (tmp_col < 0) or (tmp_col >= block_table_stride_req):
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (bt_row_idx).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, (tmp_col).to(tl.int64) + _GUARD_BIAS)
|
||||
actual_src_block_id = tl.load(
|
||||
block_table_base + tmp_col, mask=tmp_col_ok, other=-1
|
||||
).to(tl.int64)
|
||||
if actual_src_block_id <= 0:
|
||||
return
|
||||
src_addr = state_base_addr + actual_src_block_id * state_block_stride
|
||||
# Use natural block data size (inner_size * elem_size), NOT
|
||||
# state_block_stride which is the page stride and can exceed the
|
||||
@@ -378,6 +726,8 @@ def postprocess_mamba_fused_kernel(
|
||||
# block table.
|
||||
block_table_ptrs_ptr,
|
||||
block_table_stride_req: tl.int64, # stride between requests (in elements)
|
||||
block_table_num_rows: tl.int64, # bounds-guard instrumentation
|
||||
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||
# Mamba state metadata, flattened across all layer/state tensors. Different
|
||||
# Mamba groups may contribute different numbers of state tensors.
|
||||
state_base_addrs_ptr, # base address of each state tensor
|
||||
@@ -409,6 +759,14 @@ def postprocess_mamba_fused_kernel(
|
||||
# PRECOMPUTED_NEW_COMPUTED: when True, num_computed_tokens_ptr already holds
|
||||
# the post-step new_num_computed value (V2 supplies the advanced count).
|
||||
PRECOMPUTED_NEW_COMPUTED: tl.constexpr = False,
|
||||
# BT_ROW_IS_REQ: when True, block_table_ptrs_ptr points at the *persistent*
|
||||
# per-req-slot block tables (BlockTables.block_tables[i].gpu,
|
||||
# block_table.py:53-59), whose rows are request state slots, so the row
|
||||
# index is req_idx rather than batch_idx. When False it points at the
|
||||
# batch-order gather (BlockTables.input_block_tables, block_table.py:74-76)
|
||||
# and batch_idx is the row. Required by the PP-deferred post-process, which
|
||||
# runs pp_size steps after the gather that produced its idx_mapping.
|
||||
BT_ROW_IS_REQ: tl.constexpr = False,
|
||||
# TEMPORAL_TILES: when > 1, the temporal copy body is partitioned across
|
||||
# TEMPORAL_TILES CTAs along the u64 inner range. Callers must launch a
|
||||
# 3D grid (num_reqs, total_states, TEMPORAL_TILES). Default 1 preserves
|
||||
@@ -476,7 +834,15 @@ def postprocess_mamba_fused_kernel(
|
||||
if src_block_idx == dest_block_idx and accept_token_bias == 0:
|
||||
return
|
||||
|
||||
bt_row_idx = batch_idx if HAS_IDX_MAPPING else req_idx
|
||||
if BT_ROW_IS_REQ: # noqa: SIM108 -- a ternary would drop this comment
|
||||
# Persistent per-req-slot table: the row is the request's own slot, so
|
||||
# it stays correct however many steps separate the gather from this
|
||||
# launch (the PP-deferred post-process is enqueued at the top of
|
||||
# execute_model, before this step's gather, with an idx_mapping
|
||||
# captured pp_size steps earlier).
|
||||
bt_row_idx = req_idx
|
||||
else:
|
||||
bt_row_idx = batch_idx if HAS_IDX_MAPPING else req_idx
|
||||
_copy_mamba_state_block(
|
||||
state_idx,
|
||||
bt_row_idx,
|
||||
@@ -485,6 +851,8 @@ def postprocess_mamba_fused_kernel(
|
||||
accept_token_bias,
|
||||
block_table_ptrs_ptr,
|
||||
block_table_stride_req,
|
||||
block_table_num_rows,
|
||||
guard_ctr,
|
||||
state_base_addrs_ptr,
|
||||
state_block_strides_ptr,
|
||||
state_elem_sizes_ptr,
|
||||
@@ -556,6 +924,8 @@ def precopy_mamba_align_fused_kernel(
|
||||
# Same flattened state-layout metadata as postprocess_mamba_fused_kernel
|
||||
block_table_ptrs_ptr,
|
||||
block_table_stride_req: tl.int64,
|
||||
block_table_num_rows: tl.int64, # bounds-guard instrumentation
|
||||
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||
state_base_addrs_ptr,
|
||||
state_block_strides_ptr,
|
||||
state_elem_sizes_ptr,
|
||||
@@ -616,6 +986,8 @@ def precopy_mamba_align_fused_kernel(
|
||||
token_bias,
|
||||
block_table_ptrs_ptr,
|
||||
block_table_stride_req,
|
||||
block_table_num_rows,
|
||||
guard_ctr,
|
||||
state_base_addrs_ptr,
|
||||
state_block_strides_ptr,
|
||||
state_elem_sizes_ptr,
|
||||
@@ -810,6 +1182,31 @@ class MambaSpecDecodeGPUContext:
|
||||
block_table_ptrs: torch.Tensor
|
||||
block_table_stride_req: int = 0
|
||||
|
||||
# Per-group *persistent* per-req-slot block-table addresses: int64[num_groups].
|
||||
# `block_table_ptrs` above addresses the batch-order gather
|
||||
# (BlockTables.input_block_tables, block_table.py:74-76), which is rewritten
|
||||
# in place every step and is only valid for the step that gathered it. These
|
||||
# address BlockTables.block_tables[i].gpu (block_table.py:53-59), the
|
||||
# persistent [max_num_reqs, max_num_blocks] tables the gather reads from
|
||||
# (their data_ptrs are already held device-side at block_table.py:94-96).
|
||||
# Rows there are request state slots, so they can be indexed by req_idx from
|
||||
# any later step. Used only by the align post-process (BT_ROW_IS_REQ).
|
||||
persistent_block_table_ptrs: torch.Tensor | None = None
|
||||
has_persistent_block_tables: bool = False
|
||||
# shape[0] of the persistent tables; see the note in set_block_tables.
|
||||
# -1, not 0: this is the ROW BOUND the align post-process uses, and a 0
|
||||
# default bounds EVERY row out, so a context that never registered the
|
||||
# persistent tables would report guard hits on correct metadata instead of
|
||||
# copying state. -1 is unusable on purpose and is asserted before use.
|
||||
persistent_block_table_num_rows: int = -1
|
||||
|
||||
# The same tensors `persistent_block_table_ptrs` addresses, retained so the
|
||||
# host-side slot-ownership check can READ the rows the kernel dereferences
|
||||
# rather than model them. Holding these costs no allocation: they are the
|
||||
# runner's own `BlockTables.block_tables[i].gpu`, alive for the engine's
|
||||
# lifetime, and nothing here writes to them.
|
||||
persistent_block_table_views: list[torch.Tensor] | None = None
|
||||
|
||||
# persistent output for the once-per-step, all-group aligned-index launch.
|
||||
# shape: [num_groups, max_num_reqs, 1 + num_speculative_blocks].
|
||||
aligned_state_indices: torch.Tensor | None = None
|
||||
@@ -828,6 +1225,18 @@ class MambaSpecDecodeGPUContext:
|
||||
# Flag to track if metadata has been populated
|
||||
is_initialized: bool = False
|
||||
|
||||
# Last result of the host-side slot-ownership check (always [] unless
|
||||
# VLLM_E2_MAMBA_SLOT_OWNERSHIP is set). Readable by a test so the control
|
||||
# does not have to parse log records.
|
||||
last_slot_ownership_violations: list[str] = dataclasses.field(
|
||||
default_factory=list
|
||||
)
|
||||
# Liveness for the same check (diagnostics on 0.30): how many launches it has
|
||||
# judged and how many had findings. Logged on the first check and every
|
||||
# _E2_CHECK_HEARTBEAT after, so a zero-findings run can prove it ran.
|
||||
slot_ownership_checks_run: int = 0
|
||||
slot_ownership_checks_with_findings: int = 0
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls,
|
||||
@@ -900,6 +1309,9 @@ class MambaSpecDecodeGPUContext:
|
||||
block_table_ptrs=torch.zeros(
|
||||
len(mamba_group_ids), dtype=torch.int64, device=device
|
||||
),
|
||||
persistent_block_table_ptrs=torch.zeros(
|
||||
len(mamba_group_ids), dtype=torch.int64, device=device
|
||||
),
|
||||
aligned_state_indices=torch.empty(
|
||||
(
|
||||
len(mamba_group_ids),
|
||||
@@ -924,6 +1336,7 @@ class MambaSpecDecodeGPUContext:
|
||||
forward_context: dict[str, Any],
|
||||
mamba_state_copy_funcs: MambaStateCopyFuncsByType,
|
||||
block_tables: list[torch.Tensor],
|
||||
persistent_block_tables: list[torch.Tensor] | None = None,
|
||||
) -> None:
|
||||
"""Extract and cache memory layout metadata from Mamba state tensors.
|
||||
|
||||
@@ -954,10 +1367,14 @@ class MambaSpecDecodeGPUContext:
|
||||
have a `kv_cache` attribute containing the list of state tensors.
|
||||
mamba_state_copy_funcs: Mapping from MambaAttentionBackendEnum to
|
||||
copy functions.
|
||||
block_tables: per-mamba-group persistent block-table tensors, in
|
||||
the same order as `mamba_group_ids`. Their `data_ptr()` /
|
||||
`stride(0)` are captured once for the kernel to index into.
|
||||
|
||||
block_tables: per-mamba-group *gathered*, batch-order block-table
|
||||
tensors, in the same order as `mamba_group_ids`. Their
|
||||
`data_ptr()` / `stride(0)` are captured once for the kernel to
|
||||
index into by batch row.
|
||||
persistent_block_tables: per-mamba-group *persistent* per-req-slot
|
||||
block-table tensors (`BlockTables.block_tables[i].gpu`), same
|
||||
order. Captured separately so the align post-process can index
|
||||
by req slot; see `persistent_block_table_ptrs`.
|
||||
"""
|
||||
if self.is_initialized:
|
||||
return
|
||||
@@ -968,6 +1385,7 @@ class MambaSpecDecodeGPUContext:
|
||||
forward_context,
|
||||
mamba_state_copy_funcs,
|
||||
block_tables,
|
||||
persistent_block_tables,
|
||||
)
|
||||
|
||||
def _populate_metadata(
|
||||
@@ -976,6 +1394,7 @@ class MambaSpecDecodeGPUContext:
|
||||
forward_context: dict[str, Any],
|
||||
mamba_state_copy_funcs: MambaStateCopyFuncsByType,
|
||||
block_tables: list[torch.Tensor],
|
||||
persistent_block_tables: list[torch.Tensor] | None = None,
|
||||
) -> None:
|
||||
idx = 0
|
||||
for group_local_idx, mamba_group_id in enumerate(self.mamba_group_ids):
|
||||
@@ -1084,9 +1503,90 @@ class MambaSpecDecodeGPUContext:
|
||||
f"all mamba block tables must share stride(0), got {strides}"
|
||||
)
|
||||
self.block_table_stride_req = int(next(iter(strides)))
|
||||
# Bounds-guard instrumentation: row bound for the guard in _copy_mamba_state_block.
|
||||
#
|
||||
# NOT from `block_tables`. Every caller passes a per-batch VIEW --
|
||||
# get_device_tensor(num_reqs) at :1690 and :1838, a num_reqs_padded
|
||||
# slice at mamba_hybrid.py:196 -- and this function runs once, so a
|
||||
# bound taken from shape[0] here freezes at the first batch's request
|
||||
# count and rejects every higher row in every later batch. The guard
|
||||
# would then skip real state copies and report masked faults that never
|
||||
# happened. (stride(0) is unaffected: it is max_num_blocks and survives
|
||||
# a row slice, which is why the existing stride assert never caught
|
||||
# this.)
|
||||
#
|
||||
# The persistent tables ARE [max_num_reqs, ...], so use them when they
|
||||
# are registered. When they are not there is no correct row count at
|
||||
# this point, and a wrong bound is worse than a loose one: fall back to
|
||||
# a sentinel that still rejects negative rows -- the case the poison
|
||||
# guard row exercises and the only one that reads outside the tensor --
|
||||
# while never rejecting a real request slot.
|
||||
rows = {bt.shape[0] for bt in block_tables}
|
||||
assert len(rows) == 1, (
|
||||
f"all mamba block tables must share shape[0], got {rows}"
|
||||
)
|
||||
self.block_table_num_rows = _UNBOUNDED_ROWS
|
||||
for i, bt in enumerate(block_tables):
|
||||
self.block_table_ptrs[i] = _reinterpret_u64_as_i64(bt.data_ptr())
|
||||
|
||||
# Same capture for the persistent per-req-slot tables. `input_block_tables`
|
||||
# is `torch.zeros_like` of these (block_table.py:74-76), so stride(0) is
|
||||
# identical and `block_table_stride_req` covers both; assert it rather
|
||||
# than assume.
|
||||
if persistent_block_tables is not None:
|
||||
assert len(persistent_block_tables) == self.num_groups, (
|
||||
f"expected {self.num_groups} persistent block tables, "
|
||||
f"got {len(persistent_block_tables)}"
|
||||
)
|
||||
persistent_strides = {bt.stride(0) for bt in persistent_block_tables}
|
||||
assert persistent_strides == strides, (
|
||||
"persistent block tables must share stride(0) with the gathered "
|
||||
f"tables, got {persistent_strides} vs {strides}"
|
||||
)
|
||||
assert self.persistent_block_table_ptrs is not None
|
||||
for i, bt in enumerate(persistent_block_tables):
|
||||
self.persistent_block_table_ptrs[i] = _reinterpret_u64_as_i64(
|
||||
bt.data_ptr()
|
||||
)
|
||||
# The align paths index the PERSISTENT tables by req_idx while
|
||||
# every other geometry scalar is measured from the GATHERED ones
|
||||
# (by design). Two different scalars, two different treatments:
|
||||
#
|
||||
# * the STRIDE is load-bearing -- it is the multiplier in
|
||||
# `block_table_typed + bt_row_idx * block_table_stride_req`, so
|
||||
# a mismatch reads the wrong row entirely. It is ASSERTED equal
|
||||
# just above; there is no correct behaviour if it differs.
|
||||
# * the ROW COUNT is not required to be equal (the fixture in
|
||||
# test_pp_deferred_mamba_block_table.py builds a 2-row gathered
|
||||
# table against an 8-row persistent one on purpose), so it is
|
||||
# CARRIED separately. Anything bounding a row against the
|
||||
# persistent tables must use this scalar; using
|
||||
# block_table_num_rows would falsely reject a valid request slot
|
||||
# whenever the gathered buffer is narrower, and a guard that
|
||||
# fires on correct metadata is worse than no guard.
|
||||
persistent_rows = {bt.shape[0] for bt in persistent_block_tables}
|
||||
assert len(persistent_rows) == 1, (
|
||||
f"all persistent block tables must share shape[0], got "
|
||||
f"{persistent_rows}"
|
||||
)
|
||||
self.persistent_block_table_num_rows = int(next(iter(persistent_rows)))
|
||||
# ...and this is the only trustworthy row count in this object, so
|
||||
# the same-step guard takes it too.
|
||||
self.block_table_num_rows = self.persistent_block_table_num_rows
|
||||
self.has_persistent_block_tables = True
|
||||
self.persistent_block_table_views = list(persistent_block_tables)
|
||||
else:
|
||||
# Legal for V1 callers, which never reach the V2 align
|
||||
# post-process. If a V2 align launch does happen on such a context,
|
||||
# `run_fused_postprocess_align` raises rather than falling back to
|
||||
# the batch-order gather: that fallback is exactly the earlier
|
||||
# wrong-row bug.
|
||||
logger.warning_once(
|
||||
"Mamba align context initialized without the persistent "
|
||||
"per-req-slot block tables. The V1 paths are unaffected; a V2 "
|
||||
"align post-process launch on this context will raise."
|
||||
)
|
||||
|
||||
self.is_initialized = True
|
||||
|
||||
def compute_aligned_state_indices(
|
||||
@@ -1168,6 +1668,10 @@ class MambaSpecDecodeGPUContext:
|
||||
num_draft_tokens_gpu,
|
||||
self.block_table_ptrs,
|
||||
self.block_table_stride_req,
|
||||
self.block_table_num_rows,
|
||||
guard_telemetry.counter(
|
||||
guard_telemetry.SITE_MAMBA_COPY, self.state_base_addrs.device
|
||||
),
|
||||
self.state_base_addrs,
|
||||
self.state_block_strides,
|
||||
self.state_elem_sizes,
|
||||
@@ -1215,6 +1719,10 @@ class MambaSpecDecodeGPUContext:
|
||||
token_bias_gpu,
|
||||
self.block_table_ptrs,
|
||||
self.block_table_stride_req,
|
||||
self.block_table_num_rows,
|
||||
guard_telemetry.counter(
|
||||
guard_telemetry.SITE_MAMBA_COPY, self.state_base_addrs.device
|
||||
),
|
||||
self.state_base_addrs,
|
||||
self.state_block_strides,
|
||||
self.state_elem_sizes,
|
||||
@@ -1252,11 +1760,49 @@ class MambaSpecDecodeGPUContext:
|
||||
if num_reqs == 0 or not self.is_initialized:
|
||||
return
|
||||
|
||||
# Fail closed. Under PP this launch is the *deferred* post-process: it
|
||||
# is enqueued by `update_pp_decode_requests` at the top of
|
||||
# `execute_model` (model_runner.py:1644), before that step's
|
||||
# `prepare_attn` (model_runner.py:1718) and the
|
||||
# `gather_block_tables` inside it (:1442 / block_table.py:149), and
|
||||
# `idx_mapping` was captured pp_size steps earlier
|
||||
# (pp_utils.py:57-58, 258-271). The single batch-order gather buffer
|
||||
# therefore holds a different micro-batch's rows and `batch_idx`
|
||||
# resolves to the wrong request. This path MUST index the persistent
|
||||
# per-req-slot tables by `req_idx`; there is no safe fallback, so a
|
||||
# context without them is a bug in the caller, not a degraded mode.
|
||||
# (`req_idx` is also correct on the same-step last-rank / no-PP path.)
|
||||
if not self.has_persistent_block_tables:
|
||||
raise RuntimeError(
|
||||
"MambaSpecDecodeGPUContext was initialized without the "
|
||||
"persistent per-req-slot block tables, so the V2 align "
|
||||
"post-process cannot index them by req slot. Falling back to "
|
||||
"the batch-order gather is the known deferred-PP wrong-row "
|
||||
"corruption, so this raises instead. The runner must call "
|
||||
"ModelState.set_block_tables() before the first align step."
|
||||
)
|
||||
block_table_ptrs = self.persistent_block_table_ptrs
|
||||
assert self.persistent_block_table_num_rows > 0, (
|
||||
"persistent_block_table_num_rows was never set, so the row bound "
|
||||
"in _copy_mamba_state_block would reject every row and report "
|
||||
"guard hits on correct metadata. set_block_tables() must run "
|
||||
"before the first align post-process."
|
||||
)
|
||||
|
||||
# V2 reads non-contiguous idx_mapping positions, so snapshot the whole
|
||||
# decision buffer rather than only [:num_reqs].
|
||||
num_accepted_tokens_snapshot = self.num_accepted_tokens_out
|
||||
num_accepted_tokens_snapshot.copy_(num_accepted_tokens_gpu)
|
||||
|
||||
if _SLOT_OWNERSHIP_MODE != "off":
|
||||
self._check_slot_ownership(
|
||||
num_reqs=num_reqs,
|
||||
num_accepted_tokens=num_accepted_tokens_snapshot,
|
||||
state_idx_gpu=state_idx_gpu,
|
||||
new_num_computed_tokens_gpu=new_num_computed_tokens_gpu,
|
||||
idx_mapping=idx_mapping,
|
||||
)
|
||||
|
||||
total_states = self.num_states
|
||||
grid = (num_reqs, total_states, _TEMPORAL_TILES)
|
||||
postprocess_mamba_fused_kernel[grid](
|
||||
@@ -1265,8 +1811,12 @@ class MambaSpecDecodeGPUContext:
|
||||
None, # num_scheduled: unused under PRECOMPUTED_NEW_COMPUTED
|
||||
new_num_computed_tokens_gpu,
|
||||
None, # num_draft: unused under PRECOMPUTED_NEW_COMPUTED
|
||||
self.block_table_ptrs,
|
||||
block_table_ptrs,
|
||||
self.block_table_stride_req,
|
||||
self.persistent_block_table_num_rows,
|
||||
guard_telemetry.counter(
|
||||
guard_telemetry.SITE_MAMBA_COPY, self.state_base_addrs.device
|
||||
),
|
||||
self.state_base_addrs,
|
||||
self.state_block_strides,
|
||||
self.state_elem_sizes,
|
||||
@@ -1283,9 +1833,136 @@ class MambaSpecDecodeGPUContext:
|
||||
CONV_STATE_DIM_FIRST=is_conv_state_dim_first(),
|
||||
HAS_IDX_MAPPING=True,
|
||||
PRECOMPUTED_NEW_COMPUTED=True,
|
||||
BT_ROW_IS_REQ=ALIGN_BT_ROW_IS_REQ,
|
||||
TEMPORAL_TILES=_TEMPORAL_TILES,
|
||||
)
|
||||
|
||||
def _check_slot_ownership(
|
||||
self,
|
||||
*,
|
||||
num_reqs: int,
|
||||
num_accepted_tokens: torch.Tensor,
|
||||
state_idx_gpu: torch.Tensor,
|
||||
new_num_computed_tokens_gpu: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
) -> list[str]:
|
||||
"""Host-side mamba state-slot ownership check, off by default.
|
||||
|
||||
VLLM_E2_MAMBA_SLOT_OWNERSHIP=warn|raise. Same discipline as the KV
|
||||
zeroer's ownership check (VLLM_E2_ZERO_OWNERSHIP_CHECK): a diagnostic in the
|
||||
scheduler-visible sense -- host side, opt-in, never a device guard.
|
||||
Unlike that check it lives here, because the thing at risk is not visible to
|
||||
the scheduler at all: `MambaSpec` is not an `AttentionSpec`, so mamba
|
||||
state blocks never appear on `new_block_ids_to_zero` and that check never
|
||||
sees them.
|
||||
|
||||
COST, and it is not only wall-clock. Four device->host copies plus one
|
||||
block-table slice per call, each a synchronize. That SERIALIZES work
|
||||
that is normally overlapped, so an instrumented boot has materially
|
||||
different timing from a production one: zero findings under this check
|
||||
cannot by itself exclude a timing-sensitive defect. Run the
|
||||
instrumented arm for diagnosis and an uninstrumented arm for the
|
||||
concurrency qualification; do not merge their conclusions.
|
||||
|
||||
CAPTURE CONTRACT, stated explicitly rather than assumed. These copies
|
||||
are illegal inside a stream capture, so the function asserts it is not
|
||||
capturing instead of relying on "this is engine-thread Python" -- Python
|
||||
also runs while a graph is being captured, and the caller's own contract
|
||||
is the thing being checked here. A captured REPLAY does not re-enter
|
||||
this function at all, which is a real scope limit of the instrument,
|
||||
not a safety argument.
|
||||
|
||||
Returns the violation list (also stored on the context so a test can
|
||||
read it without parsing logs).
|
||||
"""
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
# The D2H copies below would abort the capture. Report rather than
|
||||
# silently skip: a diagnostic reached from a path it cannot serve
|
||||
# is a contract error in the caller.
|
||||
logger.warning_once(
|
||||
"VLLM_E2_MAMBA_SLOT_OWNERSHIP is set but the align "
|
||||
"post-process was reached during a cudagraph capture, where "
|
||||
"the check's device->host copies are illegal. Skipped for "
|
||||
"this launch; the caller's outside-capture contract does not "
|
||||
"hold."
|
||||
)
|
||||
self.last_slot_ownership_violations = []
|
||||
return []
|
||||
tables = self.persistent_block_table_views
|
||||
if not tables:
|
||||
# Unreachable from run_fused_postprocess_align, which raises above
|
||||
# when the persistent tables were never registered. Kept so the
|
||||
# check degrades to silence rather than to an exception if some
|
||||
# future caller reaches it first.
|
||||
logger.warning_once(
|
||||
"VLLM_E2_MAMBA_SLOT_OWNERSHIP is set but the persistent "
|
||||
"per-req-slot block tables were never registered, so slot "
|
||||
"ownership cannot be checked."
|
||||
)
|
||||
self.last_slot_ownership_violations = []
|
||||
return []
|
||||
|
||||
num_cols = min(int(t.shape[1]) for t in tables)
|
||||
violations = mamba_slot_ownership_violations(
|
||||
num_reqs=int(num_reqs),
|
||||
idx_mapping=idx_mapping.detach().to("cpu").reshape(-1).tolist(),
|
||||
bt_row_is_req=bool(ALIGN_BT_ROW_IS_REQ),
|
||||
block_tables=[
|
||||
t[:, :num_cols].detach().to("cpu").tolist() for t in tables
|
||||
],
|
||||
num_rows=int(self.persistent_block_table_num_rows),
|
||||
# The independent evidence: what the kernel was handed, against
|
||||
# what is being read here. `persistent_block_table_ptrs` holds the
|
||||
# addresses reinterpreted as signed, so undo that before comparing.
|
||||
registered_ptrs=(
|
||||
[
|
||||
_reinterpret_i64_as_u64(int(v))
|
||||
for v in self.persistent_block_table_ptrs.detach()
|
||||
.to("cpu")
|
||||
.reshape(-1)
|
||||
.tolist()[: len(tables)]
|
||||
]
|
||||
if self.persistent_block_table_ptrs is not None
|
||||
else None
|
||||
),
|
||||
view_ptrs=[int(t.data_ptr()) for t in tables],
|
||||
num_accepted_tokens=num_accepted_tokens.detach()
|
||||
.to("cpu")
|
||||
.reshape(-1)
|
||||
.tolist(),
|
||||
state_idx=state_idx_gpu.detach().to("cpu").reshape(-1).tolist(),
|
||||
new_num_computed_tokens=new_num_computed_tokens_gpu.detach()
|
||||
.to("cpu")
|
||||
.reshape(-1)
|
||||
.tolist(),
|
||||
block_size=int(self.block_size),
|
||||
)
|
||||
self.last_slot_ownership_violations = violations
|
||||
self.slot_ownership_checks_run += 1
|
||||
if violations:
|
||||
self.slot_ownership_checks_with_findings += 1
|
||||
runs = self.slot_ownership_checks_run
|
||||
if runs == 1 or runs % _E2_CHECK_HEARTBEAT == 0:
|
||||
logger.info(
|
||||
"Mamba slot-ownership check (mode=%s): %d checks run, %d with "
|
||||
"findings",
|
||||
_SLOT_OWNERSHIP_MODE,
|
||||
runs,
|
||||
self.slot_ownership_checks_with_findings,
|
||||
)
|
||||
if not violations:
|
||||
return violations
|
||||
message = (
|
||||
f"Mamba state-slot ownership violation on the align "
|
||||
f"post-process ({len(violations)} finding(s), num_reqs="
|
||||
f"{num_reqs}, BT_ROW_IS_REQ={ALIGN_BT_ROW_IS_REQ}): "
|
||||
+ " | ".join(violations)
|
||||
)
|
||||
if _SLOT_OWNERSHIP_MODE == "raise":
|
||||
raise RuntimeError(message)
|
||||
logger.warning("%s", message)
|
||||
return violations
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class MambaBuffers:
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""v2.2 ring-row tag fix: worker side of the ring-row tag clear.
|
||||
|
||||
The scheduler reports every newly claimed ring (``CircularBufferSpec``) block
|
||||
in ``SchedulerOutput.new_ring_block_ids_to_clear``, whether or not KV zeroing
|
||||
is on. Before the step's forward, the runner hands them to every layer that
|
||||
keeps a tagged ring (``clear_ring_row_tags``; today ``QSAKeyStateCache``),
|
||||
which zeroes only the tag word of each row. A zero tag never validates, so the
|
||||
new owner cannot pool a row it did not commit itself, whatever the previous
|
||||
occupant (or another group that held the block) left there.
|
||||
|
||||
ORDER. The runners call this right after the KV zeroer and before the CoW
|
||||
copies and the forward, on the same stream, inside the same execute_model as
|
||||
the claim's first step. So: previous owner's work done (deferred free) ->
|
||||
claim -> tag clear -> new owner's first read/write.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def ring_tag_layers(static_forward_context: Mapping[str, object]) -> list:
|
||||
"""Layers of this worker that keep a tagged ring."""
|
||||
return [
|
||||
layer
|
||||
for layer in static_forward_context.values()
|
||||
if callable(getattr(layer, "clear_ring_row_tags", None))
|
||||
]
|
||||
|
||||
|
||||
def clear_ring_row_tags(
|
||||
layers: Iterable,
|
||||
block_ids: list[int],
|
||||
device: torch.device,
|
||||
) -> int:
|
||||
"""Zero the tag word of every row of ``block_ids`` in every layer.
|
||||
|
||||
Returns the number of layers cleared. Raises (fail loud) on a block id
|
||||
outside a layer's pool: an index_fill_ with a bad index is a device-side
|
||||
assert, far from its cause."""
|
||||
if not block_ids:
|
||||
return 0
|
||||
layers = list(layers)
|
||||
if not layers:
|
||||
return 0
|
||||
lo, hi = min(block_ids), max(block_ids)
|
||||
index = torch.tensor(block_ids, dtype=torch.long, device=device)
|
||||
for layer in layers:
|
||||
tags = getattr(layer, "ring_tag_cache", None)
|
||||
if tags is None:
|
||||
raise RuntimeError(
|
||||
f"ring tag clear: layer {getattr(layer, 'prefix', layer)!r} has "
|
||||
"no bound ring_tag_cache"
|
||||
)
|
||||
if lo < 0 or hi >= tags.shape[0]:
|
||||
raise ValueError(
|
||||
f"ring tag clear: block ids [{lo}, {hi}] outside the pool of "
|
||||
f"{tags.shape[0]} blocks ({getattr(layer, 'prefix', layer)!r})"
|
||||
)
|
||||
layer.clear_ring_row_tags(index)
|
||||
return len(layers)
|
||||
@@ -39,10 +39,17 @@ from vllm.v1.kv_cache_interface import (
|
||||
UniformTypeKVCacheSpecs,
|
||||
create_kv_cache_views,
|
||||
)
|
||||
from vllm.v1.worker import guard_telemetry
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Bias so a negative row/index survives atomic_max on a zeroed counter
|
||||
# -- it MUST be a tl.constexpr: a plain module-level int is not readable
|
||||
# from a @triton.jit body -- an earlier build shipped one and every guarded kernel died
|
||||
# with "NameError: Cannot access global variable _GUARD_BIAS" on the smoke.
|
||||
_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS)
|
||||
|
||||
|
||||
def raise_if_nan_logits(num_nans_in_logits: Mapping[str, int]) -> None:
|
||||
if not any(num_nans_in_logits.values()):
|
||||
@@ -62,6 +69,8 @@ def _zero_kv_blocks_kernel(
|
||||
seg_block_strides_ptr,
|
||||
seg_page_sizes_ptr,
|
||||
block_ids_ptr,
|
||||
num_pool_blocks, # bounds-guard instrumentation
|
||||
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
"""Zero KV cache blocks across all segments in a single launch.
|
||||
@@ -91,6 +100,17 @@ def _zero_kv_blocks_kernel(
|
||||
if chunk_offset >= page_size_el:
|
||||
return
|
||||
block_id = tl.load(block_ids_ptr + block_index)
|
||||
# Bounds-guard instrumentation (guard 5): block_id goes straight into
|
||||
# block_id * block_stride_el with no pool bound, and this kernel masks only
|
||||
# WITHIN a segment page, so a corrupt id writes zeros elsewhere in the KV
|
||||
# allocation instead of faulting visibly. A valid pool id is always in
|
||||
# range (every group's tensor is asserted to be an exact multiple of
|
||||
# kv_cache_config.num_blocks), so this can only fire on corruption.
|
||||
if (block_id < 0) or (block_id >= num_pool_blocks):
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (block_index).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, (block_id).to(tl.int64) + _GUARD_BIAS)
|
||||
return
|
||||
seg_addr = tl.load(seg_addrs_ptr + seg_index)
|
||||
ptr = tl.cast(seg_addr, tl.pointer_type(tl.int32))
|
||||
block_offset = block_id.to(tl.int64) * block_stride_el.to(tl.int64)
|
||||
@@ -118,6 +138,7 @@ class KVBlockZeroer:
|
||||
static_forward_context: dict[str, Any],
|
||||
num_blocks: int,
|
||||
runner_only_attn_layers: set[str] | None = None,
|
||||
skip_kv_cache_group_ids: Iterable[int] = (),
|
||||
) -> None:
|
||||
"""Precompute the absolute-address table for the Triton zeroing kernel.
|
||||
|
||||
@@ -136,8 +157,17 @@ class KVBlockZeroer:
|
||||
physical block stride and zeroed page span remain independent.
|
||||
|
||||
Only AttentionSpec layers are processed; Mamba layers are skipped.
|
||||
|
||||
``skip_kv_cache_group_ids`` (v2.5 side-cache layout): slot-packed ring
|
||||
groups, whose views hold several slots per pool block. Their claims
|
||||
never enter whole-block zeroing (the ring-tag path clears their tag words instead),
|
||||
and with one view block per pool block their pages sat inside the
|
||||
first attention page, so skipping them leaves the zeroed bytes of
|
||||
every block unchanged.
|
||||
"""
|
||||
self.device = device
|
||||
# Bounds-guard instrumentation: pool bound for the guard in _zero_kv_blocks_kernel.
|
||||
self._num_pool_blocks = int(num_blocks)
|
||||
self._meta: (
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int] | None
|
||||
) = None
|
||||
@@ -152,12 +182,15 @@ class KVBlockZeroer:
|
||||
seg_block_strides: list[int] = []
|
||||
seg_page_sizes: list[int] = []
|
||||
|
||||
skip_group_ids = set(skip_kv_cache_group_ids)
|
||||
for group in attn_groups_iter:
|
||||
spec = group.kv_cache_spec
|
||||
if not isinstance(spec, AttentionSpec):
|
||||
continue
|
||||
if group.kv_cache_group_id >= len(kernel_block_sizes):
|
||||
continue
|
||||
if group.kv_cache_group_id in skip_group_ids:
|
||||
continue
|
||||
kernel_bs = kernel_block_sizes[group.kv_cache_group_id]
|
||||
assert spec.block_size % kernel_bs == 0
|
||||
for layer_name in group.layer_names:
|
||||
@@ -246,6 +279,10 @@ class KVBlockZeroer:
|
||||
seg_block_strides,
|
||||
seg_page_sizes,
|
||||
idx,
|
||||
self._num_pool_blocks,
|
||||
guard_telemetry.counter(
|
||||
guard_telemetry.SITE_KV_ZEROER, self.device
|
||||
),
|
||||
BLOCK_SIZE=blk_size,
|
||||
)
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import inspect
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar
|
||||
|
||||
@@ -24,18 +25,34 @@ from vllm.v1.kv_cache_interface import KVCacheSpec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.outputs import AsyncModelRunnerOutput, ModelRunnerOutput
|
||||
from vllm.v1.outputs import AsyncModelRunnerOutput, DraftTokenIds, ModelRunnerOutput
|
||||
else:
|
||||
SchedulerOutput = object
|
||||
GrammarOutput = object
|
||||
AsyncModelRunnerOutput = object
|
||||
ModelRunnerOutput = object
|
||||
DraftTokenIds = object
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_R = TypeVar("_R")
|
||||
|
||||
|
||||
def _callable_accepts_kwarg(func: Callable[..., Any], name: str) -> bool:
|
||||
"""Return True if ``func`` can be called with ``name`` as a keyword."""
|
||||
try:
|
||||
params = inspect.signature(func).parameters
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
param = params.get(name)
|
||||
if param is not None:
|
||||
return param.kind in (
|
||||
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
||||
inspect.Parameter.KEYWORD_ONLY,
|
||||
)
|
||||
return any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
|
||||
|
||||
|
||||
class CompilationTimes(NamedTuple):
|
||||
language_model: float
|
||||
encoder: float
|
||||
@@ -352,6 +369,20 @@ class WorkerWrapperBase:
|
||||
def __getattr__(self, attr: str):
|
||||
return getattr(self.worker, attr)
|
||||
|
||||
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||
"""Forward draft-token lookup, dropping ``step_id`` if unsupported.
|
||||
|
||||
Plugin workers are documented to implement no-argument
|
||||
``take_draft_token_ids()``. Passing ``step_id`` to those methods would
|
||||
raise TypeError; fall back to the legacy call instead.
|
||||
"""
|
||||
method = getattr(self.worker, "take_draft_token_ids", None)
|
||||
if method is None:
|
||||
return None
|
||||
if step_id is not None and _callable_accepts_kwarg(method, "step_id"):
|
||||
return method(step_id=step_id)
|
||||
return method()
|
||||
|
||||
def _apply_mm_cache(self, scheduler_output: SchedulerOutput) -> None:
|
||||
mm_cache = self.mm_receiver_cache
|
||||
if mm_cache is None:
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Minimal PLE child bootstrap; intentionally imports no torch or vLLM."""
|
||||
|
||||
import importlib.abc
|
||||
import importlib.machinery
|
||||
import os
|
||||
import pickle
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
from multiprocessing.connection import Connection
|
||||
from types import ModuleType
|
||||
from typing import Any
|
||||
|
||||
_FAULT = None
|
||||
if os.environ.get("VLLM_PLE_TEST_FAULT") is not None:
|
||||
from vllm_ple_test_fault import (
|
||||
_FAULT as _FAULT,
|
||||
)
|
||||
from vllm_ple_test_fault import (
|
||||
_block_on_harness_gate,
|
||||
)
|
||||
|
||||
|
||||
def _send_partial_ready_frame(fd: int) -> tuple[int, int]:
|
||||
"""Write the complete header and an incomplete body in one syscall."""
|
||||
body = pickle.dumps(
|
||||
{"status": "READY", "layer_names": []},
|
||||
protocol=pickle.HIGHEST_PROTOCOL,
|
||||
)
|
||||
frame = struct.pack("!i", len(body)) + body
|
||||
fragment = frame[: 4 + max(1, len(body) // 2)]
|
||||
try:
|
||||
return os.write(fd, fragment), len(frame)
|
||||
except OSError:
|
||||
return 0, len(frame)
|
||||
|
||||
|
||||
def _send_partial_ready_header(fd: int) -> tuple[int, int]:
|
||||
"""Write fewer than the four required header bytes in one syscall."""
|
||||
header = struct.pack("!i", 1)
|
||||
try:
|
||||
return os.write(fd, header[:2]), len(header)
|
||||
except OSError:
|
||||
return 0, len(header)
|
||||
|
||||
|
||||
def _read_exact(fd: int, size: int) -> bytes:
|
||||
chunks = bytearray()
|
||||
while len(chunks) < size:
|
||||
chunk = os.read(fd, size - len(chunks))
|
||||
if not chunk:
|
||||
raise EOFError("PLE launch payload was truncated")
|
||||
chunks.extend(chunk)
|
||||
return bytes(chunks)
|
||||
|
||||
|
||||
def _recv_launch_payload(fd: int) -> dict:
|
||||
size = struct.unpack("!Q", _read_exact(fd, 8))[0]
|
||||
payload = pickle.loads(_read_exact(fd, size))
|
||||
if not isinstance(payload, dict):
|
||||
raise RuntimeError("invalid PLE bootstrap payload")
|
||||
return payload
|
||||
|
||||
|
||||
def _fires(
|
||||
cut: str,
|
||||
*,
|
||||
unit: str = "bootstrap-launch",
|
||||
transaction: str = "launch:1",
|
||||
bytes_written: int | None = None,
|
||||
frame_size: int | None = None,
|
||||
) -> bool:
|
||||
return _FAULT is not None and _FAULT.fires(
|
||||
cut,
|
||||
role="bootstrap",
|
||||
unit=unit,
|
||||
eligible=True,
|
||||
transaction_id=transaction,
|
||||
correlation_id="launch:1",
|
||||
bytes_written=bytes_written,
|
||||
frame_size=frame_size,
|
||||
)
|
||||
|
||||
|
||||
def _write_and_ack_partial(
|
||||
fd: int,
|
||||
cut: str,
|
||||
sender: Any,
|
||||
) -> bool:
|
||||
"""A short or failed prefix write cannot acknowledge or enter the park."""
|
||||
if _FAULT is None or _FAULT.cut != cut:
|
||||
# Disarmed or armed for another cut: the control stream is untouched.
|
||||
return False
|
||||
written, frame_size = sender(fd)
|
||||
if written <= 0 or written >= frame_size:
|
||||
return False
|
||||
expected = 2 if cut == "launch_partial_ready_header" else None
|
||||
if expected is not None and written != expected:
|
||||
return False
|
||||
if expected is None and written <= 4:
|
||||
return False
|
||||
return _fires(
|
||||
cut,
|
||||
bytes_written=written,
|
||||
frame_size=frame_size,
|
||||
)
|
||||
|
||||
|
||||
class _FaultImportLoader(importlib.abc.Loader):
|
||||
def __init__(self, delegate: importlib.abc.Loader) -> None:
|
||||
self._delegate = delegate
|
||||
|
||||
def create_module(self, spec):
|
||||
create = getattr(self._delegate, "create_module", None)
|
||||
return None if create is None else create(spec)
|
||||
|
||||
def exec_module(self, module: ModuleType) -> None:
|
||||
if _fires(
|
||||
"launch_block_worker_import",
|
||||
unit="bootstrap-import",
|
||||
transaction="module:vllm.v1.ple_offload.worker",
|
||||
):
|
||||
_block_on_harness_gate()
|
||||
self._delegate.exec_module(module)
|
||||
|
||||
|
||||
class _FaultImportFinder(importlib.abc.MetaPathFinder):
|
||||
_TARGET = "vllm.v1.ple_offload.worker"
|
||||
|
||||
def find_spec(self, fullname: str, path, target=None):
|
||||
if fullname != self._TARGET:
|
||||
return None
|
||||
spec = importlib.machinery.PathFinder.find_spec(fullname, path, target)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ImportError(f"cannot find {fullname}")
|
||||
if not isinstance(spec.loader, importlib.abc.Loader):
|
||||
raise ImportError(f"{fullname} has no executable loader")
|
||||
spec.loader = _FaultImportLoader(spec.loader)
|
||||
return spec
|
||||
|
||||
|
||||
def _install_import_gate() -> None:
|
||||
target = _FaultImportFinder._TARGET
|
||||
if target in sys.modules:
|
||||
raise RuntimeError("PLE worker import gate was a cache hit")
|
||||
sys.meta_path.insert(0, _FaultImportFinder())
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Receive launch state before importing torch or vLLM."""
|
||||
if len(sys.argv) != 3:
|
||||
raise SystemExit("expected inherited PLE control and parent-death fds")
|
||||
control_fd = int(sys.argv[1])
|
||||
death_fd = int(sys.argv[2])
|
||||
if control_fd == death_fd:
|
||||
raise RuntimeError("PLE control and parent-death fds must be distinct")
|
||||
|
||||
os.set_inheritable(control_fd, False)
|
||||
os.set_inheritable(death_fd, False)
|
||||
death_pipe = Connection(death_fd)
|
||||
|
||||
def monitor_parent() -> None:
|
||||
try: # noqa: SIM105
|
||||
death_pipe.recv()
|
||||
except (EOFError, OSError):
|
||||
# SIM105 declined: this module's contract is the docstring above --
|
||||
# its import list is the audited surface of the pre-torch child.
|
||||
pass
|
||||
os._exit(1)
|
||||
|
||||
threading.Thread(
|
||||
target=monitor_parent,
|
||||
daemon=True,
|
||||
name="PleOffloadBootstrapDeathMonitor",
|
||||
).start()
|
||||
|
||||
if _fires("launch_park_before_payload_read"):
|
||||
threading.Event().wait()
|
||||
|
||||
# An armed reconstruction payload invokes its reducer inside this loads().
|
||||
payload = _recv_launch_payload(control_fd)
|
||||
|
||||
if _write_and_ack_partial(
|
||||
control_fd,
|
||||
"launch_partial_ready",
|
||||
_send_partial_ready_frame,
|
||||
):
|
||||
threading.Event().wait()
|
||||
|
||||
if _write_and_ack_partial(
|
||||
control_fd,
|
||||
"launch_partial_ready_header",
|
||||
_send_partial_ready_header,
|
||||
):
|
||||
threading.Event().wait()
|
||||
|
||||
if _fires("launch_silent_ready"):
|
||||
threading.Event().wait()
|
||||
|
||||
ready_pipe = Connection(control_fd)
|
||||
vllm_config = pickle.loads(payload["vllm_config_payload"])
|
||||
|
||||
if _FAULT is not None and _FAULT.cut == "launch_block_worker_import":
|
||||
_install_import_gate()
|
||||
from vllm.v1.ple_offload.worker import PleOffloadWorker
|
||||
|
||||
PleOffloadWorker.proc_main(
|
||||
vllm_config=vllm_config,
|
||||
num_workers=int(payload["num_workers"]),
|
||||
ipc_addr=str(payload["ipc_addr"]),
|
||||
ready_pipe=ready_pipe,
|
||||
)
|
||||
@@ -0,0 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fresh-interpreter entry point for the PLE CPU-offload worker."""
|
||||
|
||||
from vllm_ple_offload_bootstrap import main
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,380 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Stdlib-only, test-only PLE fault injection.
|
||||
|
||||
This package intentionally lives outside ``vllm`` so the offload bootstrap can
|
||||
import the same singleton without pre-populating part of the vLLM package tree.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
_ENV_NAME = "VLLM_PLE_TEST_FAULT"
|
||||
_LOG_ENV_NAME = "VLLM_PLE_TEST_FAULT_LOG"
|
||||
_UINT_RE = re.compile(r"(?:0|[1-9][0-9]*)\Z")
|
||||
_POSITIVE_INT_RE = re.compile(r"[1-9][0-9]*\Z")
|
||||
_TOKEN_RE = re.compile(r"[A-Za-z0-9_.:/@+-]+\Z")
|
||||
_MAX_RECORD_BYTES = 512
|
||||
|
||||
# cut -> (role, counting unit, TP0-only)
|
||||
_CUT_META = {
|
||||
"exc_before_prepare": ("gpu", "runner-forward", False),
|
||||
"exc_after_prepare": ("gpu", "runner-forward", False),
|
||||
"exc_in_release": ("gpu", "connector-release", False),
|
||||
"exc_after_release": ("gpu", "runner-forward", False),
|
||||
"exc_async_output": ("gpu", "runner-output-relay", False),
|
||||
"drop_request": ("gpu", "connector-request", True),
|
||||
"block_input_completion": ("gpu", "connector-request", True),
|
||||
"park_before_publish": ("offload", "offload-request", False),
|
||||
"park_before_publish_ack": ("offload", "offload-request", False),
|
||||
"launch_park_before_payload_read": (
|
||||
"bootstrap",
|
||||
"bootstrap-launch",
|
||||
False,
|
||||
),
|
||||
"launch_partial_ready": ("bootstrap", "bootstrap-launch", False),
|
||||
"launch_partial_ready_header": (
|
||||
"bootstrap",
|
||||
"bootstrap-launch",
|
||||
False,
|
||||
),
|
||||
"launch_silent_ready": ("bootstrap", "bootstrap-launch", False),
|
||||
"launch_block_reconstruction": (
|
||||
"bootstrap",
|
||||
"bootstrap-reconstruction",
|
||||
False,
|
||||
),
|
||||
"launch_block_worker_import": (
|
||||
"bootstrap",
|
||||
"bootstrap-import",
|
||||
False,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _pid_start_time(pid: int | None = None) -> int:
|
||||
"""Return Linux /proc stat field 22 without trusting the comm field."""
|
||||
if pid is None:
|
||||
pid = os.getpid()
|
||||
with open(f"/proc/{pid}/stat", encoding="ascii") as stream:
|
||||
stat = stream.read()
|
||||
close_paren = stat.rfind(")")
|
||||
if close_paren < 0:
|
||||
raise RuntimeError("invalid /proc stat record")
|
||||
fields_from_three = stat[close_paren + 2 :].split()
|
||||
if len(fields_from_three) <= 19:
|
||||
raise RuntimeError("truncated /proc stat record")
|
||||
value = fields_from_three[19]
|
||||
if _POSITIVE_INT_RE.fullmatch(value) is None:
|
||||
raise RuntimeError("invalid /proc start time")
|
||||
return int(value)
|
||||
|
||||
|
||||
def _record_token(value: str | int | None) -> str:
|
||||
if value is None:
|
||||
return "-"
|
||||
token = str(value).replace("\n", "\\n").replace(" ", "_")[:160]
|
||||
if not token or _TOKEN_RE.fullmatch(token) is None:
|
||||
raise ValueError("fault diagnostic fields must be non-empty tokens")
|
||||
return token
|
||||
|
||||
|
||||
def _block_on_harness_gate() -> None:
|
||||
"""Wait in a FIFO read that releases the GIL."""
|
||||
path = os.environ.get("VLLM_PLE_TEST_FAULT_GATE")
|
||||
if path is None or not path:
|
||||
raise RuntimeError(
|
||||
"VLLM_PLE_TEST_FAULT_GATE is required for this bootstrap cut"
|
||||
)
|
||||
fd = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
if os.read(fd, 1) == b"":
|
||||
raise EOFError("PLE bootstrap fault gate closed")
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FaultState:
|
||||
count: int = 0
|
||||
fired: bool = False
|
||||
diagnostic_failed: bool = False
|
||||
lock: threading.Lock = field(default_factory=threading.Lock)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FaultSpec:
|
||||
"""Immutable fault selection with process-local one-shot runtime state."""
|
||||
|
||||
cut: str
|
||||
ordinal: int
|
||||
tp_rank: int | None
|
||||
pp_rank: int | None
|
||||
phase: str
|
||||
role: str
|
||||
unit: str
|
||||
log_fd: int = field(compare=False, repr=False)
|
||||
_state: _FaultState = field(
|
||||
default_factory=_FaultState,
|
||||
compare=False,
|
||||
repr=False,
|
||||
)
|
||||
|
||||
def _emit(
|
||||
self,
|
||||
*,
|
||||
count: int,
|
||||
role: str,
|
||||
unit: str,
|
||||
tp_rank: int | None,
|
||||
pp_rank: int | None,
|
||||
dp_rank: int | None,
|
||||
transaction_id: str | int | None,
|
||||
correlation_id: str | int | None,
|
||||
bytes_written: int | None = None,
|
||||
bytes_sent: int | None = None,
|
||||
frame_size: int | None = None,
|
||||
) -> bool:
|
||||
try:
|
||||
pid = os.getpid()
|
||||
suffix = ""
|
||||
if bytes_written is not None:
|
||||
if bytes_sent is not None or frame_size is None:
|
||||
return False
|
||||
suffix = (
|
||||
f" bytes_written={int(bytes_written)} frame_size={int(frame_size)}"
|
||||
)
|
||||
elif bytes_sent is not None:
|
||||
if frame_size is None:
|
||||
return False
|
||||
suffix = f" bytes_sent={int(bytes_sent)} frame_size={int(frame_size)}"
|
||||
elif frame_size is not None:
|
||||
return False
|
||||
message = (
|
||||
f"PLE_TEST_FAULT v=1 cut={self.cut} ordinal={self.ordinal} "
|
||||
f"count={count} unit={_record_token(unit)} "
|
||||
f"role={_record_token(role)} "
|
||||
f"tp={tp_rank if tp_rank is not None else '-'} "
|
||||
f"pp={pp_rank if pp_rank is not None else '-'} "
|
||||
f"dp={dp_rank if dp_rank is not None else '-'} "
|
||||
f"pid={pid} pid_start={_pid_start_time(pid)} "
|
||||
f"transaction={_record_token(transaction_id)} "
|
||||
f"correlation={_record_token(correlation_id)} "
|
||||
f"monotonic={time.monotonic():.9f} "
|
||||
f"wall={time.time():.9f}{suffix}\n"
|
||||
).encode("ascii", "strict")
|
||||
if len(message) > _MAX_RECORD_BYTES:
|
||||
return False
|
||||
return os.write(self.log_fd, message) == len(message)
|
||||
except BaseException:
|
||||
return False
|
||||
|
||||
def fires(
|
||||
self,
|
||||
cut: str,
|
||||
*,
|
||||
role: str,
|
||||
unit: str,
|
||||
eligible: bool,
|
||||
tp_rank: int | None = None,
|
||||
pp_rank: int | None = None,
|
||||
dp_rank: int | None = None,
|
||||
transaction_id: str | int | None = None,
|
||||
correlation_id: str | int | None = None,
|
||||
bytes_written: int | None = None,
|
||||
frame_size: int | None = None,
|
||||
) -> bool:
|
||||
"""Count an eligible unit and fire only after a complete marker write."""
|
||||
if cut != self.cut or role != self.role or unit != self.unit:
|
||||
return False
|
||||
# phase=any does not make profile, warm-up, or dummy GPU work serving.
|
||||
if not eligible:
|
||||
return False
|
||||
if self.role == "gpu":
|
||||
if tp_rank is None or pp_rank is None or dp_rank is None:
|
||||
raise ValueError("GPU fault sites must provide TP, PP, and DP ranks")
|
||||
if self.tp_rank is not None and tp_rank != self.tp_rank:
|
||||
return False
|
||||
if self.pp_rank is not None and pp_rank != self.pp_rank:
|
||||
return False
|
||||
elif tp_rank is not None or pp_rank is not None:
|
||||
raise ValueError(f"{self.role} fault sites must not provide GPU ranks")
|
||||
|
||||
with self._state.lock:
|
||||
if self._state.fired or self._state.diagnostic_failed:
|
||||
return False
|
||||
self._state.count += 1
|
||||
count = self._state.count
|
||||
if count != self.ordinal:
|
||||
return False
|
||||
if not self._emit(
|
||||
count=count,
|
||||
role=role,
|
||||
unit=unit,
|
||||
tp_rank=tp_rank,
|
||||
pp_rank=pp_rank,
|
||||
dp_rank=dp_rank,
|
||||
transaction_id=transaction_id,
|
||||
correlation_id=correlation_id,
|
||||
bytes_written=bytes_written,
|
||||
frame_size=frame_size,
|
||||
):
|
||||
self._state.diagnostic_failed = True
|
||||
return False
|
||||
self._state.fired = True
|
||||
return True
|
||||
|
||||
def acknowledge_launch_send(self, *, bytes_sent: int, frame_size: int) -> bool:
|
||||
"""Emit bounded send progress only for the CPU-scoped launch-send gate."""
|
||||
if self.cut != "launch_park_before_payload_read":
|
||||
return False
|
||||
with self._state.lock:
|
||||
if self._state.diagnostic_failed:
|
||||
return False
|
||||
ok = self._emit(
|
||||
count=self.ordinal,
|
||||
role="launch-parent",
|
||||
unit="launch-send",
|
||||
tp_rank=None,
|
||||
pp_rank=None,
|
||||
dp_rank=None,
|
||||
transaction_id="launch:1",
|
||||
correlation_id="launch:1",
|
||||
bytes_sent=bytes_sent,
|
||||
frame_size=frame_size,
|
||||
)
|
||||
if not ok:
|
||||
self._state.diagnostic_failed = True
|
||||
return ok
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _FaultReconstructionPayload:
|
||||
"""Pickle wrapper whose reducer is the reconstruction fault boundary."""
|
||||
|
||||
payload: bytes
|
||||
|
||||
def __reduce__(self):
|
||||
return (_reconstruct_configuration_payload, (self.payload,))
|
||||
|
||||
|
||||
def _reconstruct_configuration_payload(payload: bytes) -> bytes:
|
||||
fault = _FAULT
|
||||
if fault is not None and fault.fires(
|
||||
"launch_block_reconstruction",
|
||||
role="bootstrap",
|
||||
unit="bootstrap-reconstruction",
|
||||
eligible=True,
|
||||
transaction_id="reducer:configuration",
|
||||
correlation_id="launch:1",
|
||||
):
|
||||
_block_on_harness_gate()
|
||||
return payload
|
||||
|
||||
|
||||
def _parse_uint(value: str, name: str, *, positive: bool = False) -> int:
|
||||
pattern = _POSITIVE_INT_RE if positive else _UINT_RE
|
||||
if pattern.fullmatch(value) is None:
|
||||
requirement = "a positive integer" if positive else "a non-negative integer"
|
||||
raise ValueError(f"{_ENV_NAME}: {name} must be {requirement}")
|
||||
return int(value)
|
||||
|
||||
|
||||
def _open_fault_log() -> int:
|
||||
path = os.environ.get(_LOG_ENV_NAME)
|
||||
if path is None or not path or path != path.strip():
|
||||
raise ValueError(
|
||||
f"{_LOG_ENV_NAME} is required and must be a non-empty path "
|
||||
f"when {_ENV_NAME} is armed"
|
||||
)
|
||||
flags = os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NONBLOCK
|
||||
try:
|
||||
return os.open(path, flags, 0o600)
|
||||
except OSError as error:
|
||||
raise ValueError(f"{_LOG_ENV_NAME}: cannot open {path!r}") from error
|
||||
|
||||
|
||||
def _parse_fault_spec(value: str | None) -> FaultSpec | None:
|
||||
"""Strictly parse the environment representation."""
|
||||
if value is None:
|
||||
return None
|
||||
if not value or value != value.strip():
|
||||
raise ValueError(
|
||||
f"{_ENV_NAME}: spec must be non-empty with no outer whitespace"
|
||||
)
|
||||
|
||||
fields: dict[str, str] = {}
|
||||
for item in value.split(","):
|
||||
if item.count("=") != 1:
|
||||
raise ValueError(f"{_ENV_NAME}: every field must be key=value")
|
||||
key, field_value = item.split("=", 1)
|
||||
if (
|
||||
not key
|
||||
or not field_value
|
||||
or key != key.strip()
|
||||
or field_value != field_value.strip()
|
||||
):
|
||||
raise ValueError(f"{_ENV_NAME}: fields must not contain whitespace")
|
||||
if key in fields:
|
||||
raise ValueError(f"{_ENV_NAME}: duplicate field {key!r}")
|
||||
fields[key] = field_value
|
||||
|
||||
allowed = {"cut", "ordinal", "rank", "tp", "pp", "phase"}
|
||||
unknown = set(fields).difference(allowed)
|
||||
if unknown:
|
||||
raise ValueError(f"{_ENV_NAME}: unknown field(s): {sorted(unknown)}")
|
||||
missing = {"cut", "ordinal"}.difference(fields)
|
||||
if missing:
|
||||
raise ValueError(f"{_ENV_NAME}: missing field(s): {sorted(missing)}")
|
||||
|
||||
cut = fields["cut"]
|
||||
if cut not in _CUT_META:
|
||||
raise ValueError(f"{_ENV_NAME}: unknown cut {cut!r}")
|
||||
ordinal = _parse_uint(fields["ordinal"], "ordinal", positive=True)
|
||||
role, unit, tp0_only = _CUT_META[cut]
|
||||
|
||||
has_any_rank = "rank" in fields
|
||||
has_specific_rank = "tp" in fields or "pp" in fields
|
||||
if has_any_rank:
|
||||
if fields["rank"] != "any":
|
||||
raise ValueError(f"{_ENV_NAME}: rank must be exactly 'any'")
|
||||
if has_specific_rank:
|
||||
raise ValueError(f"{_ENV_NAME}: rank=any cannot be combined with tp or pp")
|
||||
tp_rank = None
|
||||
pp_rank = None
|
||||
else:
|
||||
if not has_specific_rank:
|
||||
raise ValueError(f"{_ENV_NAME}: specify rank=any, tp, or pp")
|
||||
tp_rank = _parse_uint(fields["tp"], "tp") if "tp" in fields else None
|
||||
pp_rank = _parse_uint(fields["pp"], "pp") if "pp" in fields else None
|
||||
|
||||
if role in {"offload", "bootstrap"} and not has_any_rank:
|
||||
raise ValueError(
|
||||
f"{_ENV_NAME}: {role} cut {cut!r} requires rank=any; it has no TP/PP rank"
|
||||
)
|
||||
if role == "gpu":
|
||||
if pp_rank is not None and pp_rank != 0:
|
||||
raise ValueError(f"{_ENV_NAME}: cut {cut!r} exists only on PP rank zero")
|
||||
if tp0_only and tp_rank is not None and tp_rank != 0:
|
||||
raise ValueError(f"{_ENV_NAME}: cut {cut!r} exists only on TP rank zero")
|
||||
|
||||
phase = fields.get("phase", "serving")
|
||||
if phase not in {"serving", "any"}:
|
||||
raise ValueError(f"{_ENV_NAME}: phase must be 'serving' or 'any'")
|
||||
|
||||
return FaultSpec(
|
||||
cut=cut,
|
||||
ordinal=ordinal,
|
||||
tp_rank=tp_rank,
|
||||
pp_rank=pp_rank,
|
||||
phase=phase,
|
||||
role=role,
|
||||
unit=unit,
|
||||
log_fd=_open_fault_log(),
|
||||
)
|
||||
|
||||
|
||||
_FAULT = _parse_fault_spec(os.environ.get(_ENV_NAME))
|
||||
Reference in New Issue
Block a user