diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py index c7665c46b8..e3ad61e617 100644 --- a/vllm/v1/worker/gpu/block_table.py +++ b/vllm/v1/worker/gpu/block_table.py @@ -6,6 +6,7 @@ import torch from vllm.triton_utils import tl, triton from vllm.v1.attention.backends.utils import PAD_SLOT_ID +from vllm.v1.worker import guard_telemetry from vllm.v1.worker.gpu.buffer_utils import ( FusedStagedWriter, StagedWriteTensor, @@ -13,6 +14,12 @@ from vllm.v1.worker.gpu.buffer_utils import ( _load_ptr, ) +# Bias so a negative row/index survives atomic_max on a zeroed counter +# -- it MUST be a tl.constexpr: a plain module-level int is not readable +# from a @triton.jit body -- an earlier build shipped one and every guarded kernel died +# with "NameError: Cannot access global variable _GUARD_BIAS" on the smoke. +_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS) + class BlockTables: # Set by the elastic EP warmup so KV writes land in the null block. @@ -180,6 +187,9 @@ class BlockTables: self.num_blocks.gpu, self.num_blocks.gpu.stride(0), num_reqs, + guard_telemetry.counter( + guard_telemetry.SITE_BT_GATHER, idx_mapping.device + ), BLOCK_SIZE=1024, # type: ignore ) return tuple(bt[:num_reqs_padded] for bt in out) @@ -217,6 +227,10 @@ class BlockTables: positions, self.block_table_ptrs, self.block_table_strides, + self.num_blocks.gpu.stride(0), + guard_telemetry.counter( + guard_telemetry.SITE_SLOT_MAPPING, idx_mapping.device + ), self.block_sizes_tensor, self.kernel_block_sizes_tensor, self.slot_mapping_enabled, @@ -252,6 +266,7 @@ def _gather_block_tables_kernel( num_blocks_ptr, # [num_kv_cache_groups, max_num_reqs] num_blocks_stride, num_reqs, # actual number of requests (for padding) + guard_ctr, # bounds-guard instrumentation: persistent int64[3] BLOCK_SIZE: tl.constexpr, ): # kv cache group id @@ -271,8 +286,39 @@ def _gather_block_tables_kernel( return req_idx = tl.load(batch_idx_to_req_idx + batch_idx) + # Bounds-guard instrumentation (guard 3): req_idx indexes both num_blocks_ptr and the + # persistent source table, which have max_num_reqs rows. num_blocks_stride + # IS max_num_reqs, so the bound needs no new argument. Zero the destination + # row and return, mirroring the padded-row path above. + if (req_idx < 0) or (req_idx >= num_blocks_stride): + tl.atomic_add(guard_ctr, 1) + tl.atomic_max(guard_ctr + 1, (batch_idx).to(tl.int64) + _GUARD_BIAS) + tl.atomic_max(guard_ctr + 2, (req_idx).to(tl.int64) + _GUARD_BIAS) + for i in tl.range(0, max_num_blocks, BLOCK_SIZE): + offset = i + tl.arange(0, BLOCK_SIZE) + tl.store(dst_row_ptr + offset, 0, mask=offset < max_num_blocks) + return + group_num_blocks_ptr = num_blocks_ptr + group_id * num_blocks_stride num_blocks = tl.load(group_num_blocks_ptr + req_idx) + # A row count wider than the table would copy past the end of both rows. + # + # Clamp UNCONDITIONALLY and record conditionally. Doing the clamp inside + # the `if` rebound an int32 name (num_blocks, loaded from num_blocks_ptr) + # to int64 (max_num_blocks is `stride`, loaded from block_table_strides), + # and Triton requires a name to keep its type across a branch: + # "initial value for `num_blocks` is of type int32[], but the then block + # redefines it as int64[]" + # That killed an early boot in warmup, at the first real gather. The + # explicit .to(tl.int32) pins the type rather than relying on whatever the + # operands promote to. + if (num_blocks < 0) or (num_blocks > max_num_blocks): + tl.atomic_add(guard_ctr, 1) + tl.atomic_max(guard_ctr + 1, (batch_idx).to(tl.int64) + _GUARD_BIAS) + tl.atomic_max(guard_ctr + 2, (num_blocks).to(tl.int64) + _GUARD_BIAS) + num_blocks = tl.minimum( + tl.maximum(num_blocks, 0), max_num_blocks + ).to(tl.int32) src_block_table_ptr = _load_ptr(src_block_table_ptrs + group_id, tl.int32) src_row_ptr = src_block_table_ptr + req_idx * stride @@ -291,6 +337,8 @@ def _compute_slot_mappings_kernel( pos, # [num_tokens] block_table_ptrs, # [num_kv_cache_groups] block_table_strides, # [num_kv_cache_groups] + bt_num_rows, # bounds-guard instrumentation: rows in the persistent block tables + guard_ctr, # bounds-guard instrumentation: persistent int64[3] block_sizes, # [num_kv_cache_groups] kernel_block_sizes, # [num_kv_cache_groups] slot_mapping_enabled, # [num_kv_cache_groups] @@ -328,9 +376,27 @@ def _compute_slot_mappings_kernel( sharded = tl.load(dcp_sharded + group_id) req_state_idx = tl.load(idx_mapping + batch_idx) + # Bounds-guard instrumentation (guard 4): req_state_idx indexes the persistent block table + # before any bound is applied. An invalid row writes PAD_ID for this + # request rather than reading an arbitrary row. # idx_mapping == -1 marks a dummy (or CUDA-graph padding) request that owns # no blocks: never read its block-table row and emit PAD for its tokens. is_real_req = req_state_idx >= 0 + + # Bounds-guard instrumentation (guard 4): req_state_idx indexes the persistent block table + # before any bound is applied. An invalid row writes PAD_ID for this + # request rather than reading an arbitrary row. + if (req_state_idx < 0) or (req_state_idx >= bt_num_rows): + tl.atomic_add(guard_ctr, 1) + tl.atomic_max(guard_ctr + 1, (batch_idx).to(tl.int64) + _GUARD_BIAS) + tl.atomic_max(guard_ctr + 2, (req_state_idx).to(tl.int64) + _GUARD_BIAS) + bad_start = tl.load(query_start_loc + batch_idx) + bad_end = tl.load(query_start_loc + batch_idx + 1) + for i in range(bad_start, bad_end, TRITON_BLOCK_SIZE): + offset = i + tl.arange(0, TRITON_BLOCK_SIZE) + tl.store(slot_mapping_ptr + offset, PAD_ID, mask=offset < bad_end) + return + start_idx = tl.load(query_start_loc + batch_idx) end_idx = tl.load(query_start_loc + batch_idx + 1) for i in range(start_idx, end_idx, TRITON_BLOCK_SIZE):