[Worker] Validate req_idx in the block-table gather and the slot mapping
Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -6,6 +6,7 @@ import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker import guard_telemetry
|
||||
from vllm.v1.worker.gpu.buffer_utils import (
|
||||
FusedStagedWriter,
|
||||
StagedWriteTensor,
|
||||
@@ -13,6 +14,12 @@ from vllm.v1.worker.gpu.buffer_utils import (
|
||||
_load_ptr,
|
||||
)
|
||||
|
||||
# Bias so a negative row/index survives atomic_max on a zeroed counter
|
||||
# -- it MUST be a tl.constexpr: a plain module-level int is not readable
|
||||
# from a @triton.jit body -- an earlier build shipped one and every guarded kernel died
|
||||
# with "NameError: Cannot access global variable _GUARD_BIAS" on the smoke.
|
||||
_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS)
|
||||
|
||||
|
||||
class BlockTables:
|
||||
# Set by the elastic EP warmup so KV writes land in the null block.
|
||||
@@ -180,6 +187,9 @@ class BlockTables:
|
||||
self.num_blocks.gpu,
|
||||
self.num_blocks.gpu.stride(0),
|
||||
num_reqs,
|
||||
guard_telemetry.counter(
|
||||
guard_telemetry.SITE_BT_GATHER, idx_mapping.device
|
||||
),
|
||||
BLOCK_SIZE=1024, # type: ignore
|
||||
)
|
||||
return tuple(bt[:num_reqs_padded] for bt in out)
|
||||
@@ -217,6 +227,10 @@ class BlockTables:
|
||||
positions,
|
||||
self.block_table_ptrs,
|
||||
self.block_table_strides,
|
||||
self.num_blocks.gpu.stride(0),
|
||||
guard_telemetry.counter(
|
||||
guard_telemetry.SITE_SLOT_MAPPING, idx_mapping.device
|
||||
),
|
||||
self.block_sizes_tensor,
|
||||
self.kernel_block_sizes_tensor,
|
||||
self.slot_mapping_enabled,
|
||||
@@ -252,6 +266,7 @@ def _gather_block_tables_kernel(
|
||||
num_blocks_ptr, # [num_kv_cache_groups, max_num_reqs]
|
||||
num_blocks_stride,
|
||||
num_reqs, # actual number of requests (for padding)
|
||||
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
# kv cache group id
|
||||
@@ -271,8 +286,39 @@ def _gather_block_tables_kernel(
|
||||
return
|
||||
|
||||
req_idx = tl.load(batch_idx_to_req_idx + batch_idx)
|
||||
# Bounds-guard instrumentation (guard 3): req_idx indexes both num_blocks_ptr and the
|
||||
# persistent source table, which have max_num_reqs rows. num_blocks_stride
|
||||
# IS max_num_reqs, so the bound needs no new argument. Zero the destination
|
||||
# row and return, mirroring the padded-row path above.
|
||||
if (req_idx < 0) or (req_idx >= num_blocks_stride):
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (batch_idx).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, (req_idx).to(tl.int64) + _GUARD_BIAS)
|
||||
for i in tl.range(0, max_num_blocks, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
tl.store(dst_row_ptr + offset, 0, mask=offset < max_num_blocks)
|
||||
return
|
||||
|
||||
group_num_blocks_ptr = num_blocks_ptr + group_id * num_blocks_stride
|
||||
num_blocks = tl.load(group_num_blocks_ptr + req_idx)
|
||||
# A row count wider than the table would copy past the end of both rows.
|
||||
#
|
||||
# Clamp UNCONDITIONALLY and record conditionally. Doing the clamp inside
|
||||
# the `if` rebound an int32 name (num_blocks, loaded from num_blocks_ptr)
|
||||
# to int64 (max_num_blocks is `stride`, loaded from block_table_strides),
|
||||
# and Triton requires a name to keep its type across a branch:
|
||||
# "initial value for `num_blocks` is of type int32[], but the then block
|
||||
# redefines it as int64[]"
|
||||
# That killed an early boot in warmup, at the first real gather. The
|
||||
# explicit .to(tl.int32) pins the type rather than relying on whatever the
|
||||
# operands promote to.
|
||||
if (num_blocks < 0) or (num_blocks > max_num_blocks):
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (batch_idx).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, (num_blocks).to(tl.int64) + _GUARD_BIAS)
|
||||
num_blocks = tl.minimum(
|
||||
tl.maximum(num_blocks, 0), max_num_blocks
|
||||
).to(tl.int32)
|
||||
|
||||
src_block_table_ptr = _load_ptr(src_block_table_ptrs + group_id, tl.int32)
|
||||
src_row_ptr = src_block_table_ptr + req_idx * stride
|
||||
@@ -291,6 +337,8 @@ def _compute_slot_mappings_kernel(
|
||||
pos, # [num_tokens]
|
||||
block_table_ptrs, # [num_kv_cache_groups]
|
||||
block_table_strides, # [num_kv_cache_groups]
|
||||
bt_num_rows, # bounds-guard instrumentation: rows in the persistent block tables
|
||||
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||
block_sizes, # [num_kv_cache_groups]
|
||||
kernel_block_sizes, # [num_kv_cache_groups]
|
||||
slot_mapping_enabled, # [num_kv_cache_groups]
|
||||
@@ -328,9 +376,27 @@ def _compute_slot_mappings_kernel(
|
||||
sharded = tl.load(dcp_sharded + group_id)
|
||||
|
||||
req_state_idx = tl.load(idx_mapping + batch_idx)
|
||||
# Bounds-guard instrumentation (guard 4): req_state_idx indexes the persistent block table
|
||||
# before any bound is applied. An invalid row writes PAD_ID for this
|
||||
# request rather than reading an arbitrary row.
|
||||
# idx_mapping == -1 marks a dummy (or CUDA-graph padding) request that owns
|
||||
# no blocks: never read its block-table row and emit PAD for its tokens.
|
||||
is_real_req = req_state_idx >= 0
|
||||
|
||||
# Bounds-guard instrumentation (guard 4): req_state_idx indexes the persistent block table
|
||||
# before any bound is applied. An invalid row writes PAD_ID for this
|
||||
# request rather than reading an arbitrary row.
|
||||
if (req_state_idx < 0) or (req_state_idx >= bt_num_rows):
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (batch_idx).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, (req_state_idx).to(tl.int64) + _GUARD_BIAS)
|
||||
bad_start = tl.load(query_start_loc + batch_idx)
|
||||
bad_end = tl.load(query_start_loc + batch_idx + 1)
|
||||
for i in range(bad_start, bad_end, TRITON_BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, TRITON_BLOCK_SIZE)
|
||||
tl.store(slot_mapping_ptr + offset, PAD_ID, mask=offset < bad_end)
|
||||
return
|
||||
|
||||
start_idx = tl.load(query_start_loc + batch_idx)
|
||||
end_idx = tl.load(query_start_loc + batch_idx + 1)
|
||||
for i in range(start_idx, end_idx, TRITON_BLOCK_SIZE):
|
||||
|
||||
Reference in New Issue
Block a user