diff --git a/vllm/third_party/flash_linear_attention/ops/fused_recurrent.py b/vllm/third_party/flash_linear_attention/ops/fused_recurrent.py index eb08b938c2..2379fcb939 100644 --- a/vllm/third_party/flash_linear_attention/ops/fused_recurrent.py +++ b/vllm/third_party/flash_linear_attention/ops/fused_recurrent.py @@ -11,6 +11,12 @@ import torch from vllm.triton_utils import tl, triton +from vllm.v1.worker import guard_telemetry + +# Bias so a negative row/index survives atomic_max on a zeroed counter +# -- it MUST be a tl.constexpr: a plain module-level int is not +# readable from a @triton.jit body. +_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS) from .op import exp, log @@ -36,6 +42,7 @@ def fused_recurrent_gated_delta_rule_fwd_kernel( cu_seqlens, ssm_state_indices, num_accepted_tokens, + guard_ctr, # bounds-guard instrumentation: persistent int64[3] scale, N: tl.int64, # num of sequences T: tl.int64, # num of tokens @@ -106,12 +113,36 @@ def fused_recurrent_gated_delta_rule_fwd_kernel( i_t = tl.load(num_accepted_tokens + i_n).to(tl.int64) - 1 else: i_t = 0 - # Load state index and check for invalid entries - state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq + i_t).to( - tl.int64 - ) - # Skip if state index is invalid (NULL_BLOCK_ID=0) + # PR #50021: i_t is num_accepted_tokens - 1 and is otherwise + # unbounded, while ssm_state_indices has only stride_indices_seq + # columns per request. A zero accepted count gives i_t == -1, a + # read before this request's row (and before the tensor for + # i_n == 0); a stale or too-large count reads past the row. The + # state_idx <= 0 guard below rejects only non-positive values, so + # an out-of-range read returning a garbage positive int flowed into + # h0 + state_idx * stride_init_state_token and faulted the SM. + # + # This is the SAME defect as the decode-state guard, in the twin kernel. An + # earlier revision left it unguarded and dropped the upstream test's arm for it; + # it is now ported rather than caveated. + idx_in_row = (i_t >= 0) & (i_t < stride_indices_seq) + if (i_t < 0) or (i_t >= stride_indices_seq): + tl.atomic_add(guard_ctr, 1) + tl.atomic_max(guard_ctr + 1, (i_n).to(tl.int64) + _GUARD_BIAS) + tl.atomic_max(guard_ctr + 2, (i_t).to(tl.int64) + _GUARD_BIAS) + state_idx = tl.load( + ssm_state_indices + i_n * stride_indices_seq + i_t, + mask=idx_in_row, + other=0, + ).to(tl.int64) + # Skip invalid state indices without exposing uninitialized output. + # o is q.new_empty in the entry below, so the previous early return + # left this request's rows holding whatever the allocator returned. if state_idx <= 0: + zero = tl.zeros([BV], dtype=tl.float32).to(p_o.dtype.element_ty) + for _ in range(0, T): + tl.store(p_o, zero, mask=mask_v) + p_o += HV * V return p_h0 = h0 + state_idx * stride_init_state_token else: @@ -227,6 +258,9 @@ def fused_recurrent_gated_delta_rule_fwd( cu_seqlens=cu_seqlens, ssm_state_indices=ssm_state_indices, num_accepted_tokens=num_accepted_tokens, + guard_ctr=guard_telemetry.counter( + guard_telemetry.SITE_GDN_RECURRENT_STATE, q.device + ), scale=scale, N=N, T=T, diff --git a/vllm/third_party/flash_linear_attention/ops/kda.py b/vllm/third_party/flash_linear_attention/ops/kda.py index b5c3a92b65..3eab3e4021 100644 --- a/vllm/third_party/flash_linear_attention/ops/kda.py +++ b/vllm/third_party/flash_linear_attention/ops/kda.py @@ -14,6 +14,7 @@ import torch.nn as nn from vllm.model_executor.custom_op import CustomOp from vllm.triton_utils import tl, triton +from vllm.v1.worker import guard_telemetry from vllm.utils.math_utils import RCP_LN2, cdiv, next_power_of_2 from .chunk_delta_h import chunk_gated_delta_rule_fwd_h @@ -81,6 +82,12 @@ def fused_recurrent_kda_fwd( cu_seqlens=cu_seqlens, ssm_state_indices=ssm_state_indices, num_accepted_tokens=num_accepted_tokens, + # Bounds-guard instrumentation: the same kernel object as fused_recurrent.py:249, so it + # takes the same guard counter. Same site: the defect being guarded is + # the kernel's, not the caller's. + guard_ctr=guard_telemetry.counter( + guard_telemetry.SITE_GDN_RECURRENT_STATE, q.device + ), scale=scale, N=N, T=T,