[TopK] Exact restart of the medium path on candidate-buffer overflow
When the persistent top-k medium-path candidate buffer overflows, the row is restarted exactly instead of being truncated. Capture-safe counters with atomic publication and a coherent snapshot record every restart. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -586,6 +586,10 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"persistent_topk(Tensor logits, Tensor lengths, Tensor! output, "
|
||||
"Tensor workspace, int k, int max_seq_len) -> ()");
|
||||
|
||||
// Read (and optionally reset) the medium-path exact-restart counters.
|
||||
ops.def(
|
||||
"persistent_topk_exact_counters(Tensor! out, int reset, int sync) -> ()");
|
||||
|
||||
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
||||
ops.def(
|
||||
"cooperative_topk(Tensor logits, Tensor lengths, Tensor! output, "
|
||||
@@ -858,6 +862,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("top_k_per_row_prefill", TORCH_BOX(&top_k_per_row_prefill));
|
||||
ops.impl("top_k_per_row_decode", TORCH_BOX(&top_k_per_row_decode));
|
||||
ops.impl("persistent_topk", TORCH_BOX(&persistent_topk));
|
||||
ops.impl("persistent_topk_exact_counters",
|
||||
TORCH_BOX(&persistent_topk_exact_counters));
|
||||
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
||||
ops.impl("cooperative_topk", TORCH_BOX(&cooperative_topk));
|
||||
#endif
|
||||
|
||||
Reference in New Issue
Block a user