[ModelRunner V2] Bound memory for large logprobs requests (#46746)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-06-25 18:04:06 -07:00
committed by GitHub
parent ad28d605e6
commit 32bb3195f0
+18 -10
View File
@@ -9,6 +9,9 @@ from vllm.triton_utils import tl, triton
from vllm.v1.outputs import LogprobsTensors from vllm.v1.outputs import LogprobsTensors
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor
# Upper bound on the topk kernel's per-iteration gather width.
_MAX_TOPK_BLOCK = 1024
@triton.jit @triton.jit
def _topk_log_softmax_kernel( def _topk_log_softmax_kernel(
@@ -19,7 +22,7 @@ def _topk_log_softmax_kernel(
topk, topk,
vocab_size, vocab_size,
BLOCK_SIZE: tl.constexpr, BLOCK_SIZE: tl.constexpr,
PADDED_TOPK: tl.constexpr, TOPK_BLOCK_SIZE: tl.constexpr,
): ):
req_idx = tl.program_id(0).to(tl.int64) req_idx = tl.program_id(0).to(tl.int64)
row_ptr = logits_ptr + req_idx * logits_stride row_ptr = logits_ptr + req_idx * logits_stride
@@ -42,14 +45,16 @@ def _topk_log_softmax_kernel(
se += tl.sum(e) se += tl.sum(e)
lse = tl.log(se) lse = tl.log(se)
k_offset = tl.arange(0, PADDED_TOPK) for j in range(0, topk, TOPK_BLOCK_SIZE):
k_mask = k_offset < topk k_offset = j + tl.arange(0, TOPK_BLOCK_SIZE)
topk_ids = tl.load(topk_ids_ptr + req_idx * topk + k_offset, mask=k_mask, other=0) k_mask = k_offset < topk
topk_ids = tl.load(
logits = tl.load(row_ptr + topk_ids, mask=k_mask) topk_ids_ptr + req_idx * topk + k_offset, mask=k_mask, other=0
logits = logits.to(tl.float32) )
o = logits - max_val - lse logits = tl.load(row_ptr + topk_ids, mask=k_mask)
tl.store(output_ptr + req_idx * topk + k_offset, o, mask=k_mask) logits = logits.to(tl.float32)
o = logits - max_val - lse
tl.store(output_ptr + req_idx * topk + k_offset, o, mask=k_mask)
@triton.jit @triton.jit
@@ -85,6 +90,9 @@ def compute_token_logprobs(
token_ids = token_ids.to(torch.int64) token_ids = token_ids.to(torch.int64)
num_logprobs = token_ids.shape[1] num_logprobs = token_ids.shape[1]
logprobs = logits.new_empty((batch_size, num_logprobs), dtype=torch.float32) logprobs = logits.new_empty((batch_size, num_logprobs), dtype=torch.float32)
# Cap the kernel's per-iteration width so very large num_logprobs requests
# stream the gather in bounded-size chunks, avoiding excessive mem use.
topk_block_size = min(triton.next_power_of_2(num_logprobs), _MAX_TOPK_BLOCK)
_topk_log_softmax_kernel[(batch_size,)]( _topk_log_softmax_kernel[(batch_size,)](
logprobs, logprobs,
logits, logits,
@@ -93,7 +101,7 @@ def compute_token_logprobs(
num_logprobs, num_logprobs,
vocab_size, vocab_size,
BLOCK_SIZE=1024, # type: ignore BLOCK_SIZE=1024, # type: ignore
PADDED_TOPK=triton.next_power_of_2(num_logprobs), TOPK_BLOCK_SIZE=topk_block_size,
) )
return logprobs return logprobs