[GDN] Bound the recurrent state index
Both callers of the guarded kernel (fused recurrent and KDA) are updated in the same commit. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -11,6 +11,12 @@
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker import guard_telemetry
|
||||
|
||||
# Bias so a negative row/index survives atomic_max on a zeroed counter
|
||||
# -- it MUST be a tl.constexpr: a plain module-level int is not
|
||||
# readable from a @triton.jit body.
|
||||
_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS)
|
||||
|
||||
from .op import exp, log
|
||||
|
||||
@@ -36,6 +42,7 @@ def fused_recurrent_gated_delta_rule_fwd_kernel(
|
||||
cu_seqlens,
|
||||
ssm_state_indices,
|
||||
num_accepted_tokens,
|
||||
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||
scale,
|
||||
N: tl.int64, # num of sequences
|
||||
T: tl.int64, # num of tokens
|
||||
@@ -106,12 +113,36 @@ def fused_recurrent_gated_delta_rule_fwd_kernel(
|
||||
i_t = tl.load(num_accepted_tokens + i_n).to(tl.int64) - 1
|
||||
else:
|
||||
i_t = 0
|
||||
# Load state index and check for invalid entries
|
||||
state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq + i_t).to(
|
||||
tl.int64
|
||||
)
|
||||
# Skip if state index is invalid (NULL_BLOCK_ID=0)
|
||||
# PR #50021: i_t is num_accepted_tokens - 1 and is otherwise
|
||||
# unbounded, while ssm_state_indices has only stride_indices_seq
|
||||
# columns per request. A zero accepted count gives i_t == -1, a
|
||||
# read before this request's row (and before the tensor for
|
||||
# i_n == 0); a stale or too-large count reads past the row. The
|
||||
# state_idx <= 0 guard below rejects only non-positive values, so
|
||||
# an out-of-range read returning a garbage positive int flowed into
|
||||
# h0 + state_idx * stride_init_state_token and faulted the SM.
|
||||
#
|
||||
# This is the SAME defect as the decode-state guard, in the twin kernel. An
|
||||
# earlier revision left it unguarded and dropped the upstream test's arm for it;
|
||||
# it is now ported rather than caveated.
|
||||
idx_in_row = (i_t >= 0) & (i_t < stride_indices_seq)
|
||||
if (i_t < 0) or (i_t >= stride_indices_seq):
|
||||
tl.atomic_add(guard_ctr, 1)
|
||||
tl.atomic_max(guard_ctr + 1, (i_n).to(tl.int64) + _GUARD_BIAS)
|
||||
tl.atomic_max(guard_ctr + 2, (i_t).to(tl.int64) + _GUARD_BIAS)
|
||||
state_idx = tl.load(
|
||||
ssm_state_indices + i_n * stride_indices_seq + i_t,
|
||||
mask=idx_in_row,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
# Skip invalid state indices without exposing uninitialized output.
|
||||
# o is q.new_empty in the entry below, so the previous early return
|
||||
# left this request's rows holding whatever the allocator returned.
|
||||
if state_idx <= 0:
|
||||
zero = tl.zeros([BV], dtype=tl.float32).to(p_o.dtype.element_ty)
|
||||
for _ in range(0, T):
|
||||
tl.store(p_o, zero, mask=mask_v)
|
||||
p_o += HV * V
|
||||
return
|
||||
p_h0 = h0 + state_idx * stride_init_state_token
|
||||
else:
|
||||
@@ -227,6 +258,9 @@ def fused_recurrent_gated_delta_rule_fwd(
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
guard_ctr=guard_telemetry.counter(
|
||||
guard_telemetry.SITE_GDN_RECURRENT_STATE, q.device
|
||||
),
|
||||
scale=scale,
|
||||
N=N,
|
||||
T=T,
|
||||
|
||||
@@ -14,6 +14,7 @@ import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker import guard_telemetry
|
||||
from vllm.utils.math_utils import RCP_LN2, cdiv, next_power_of_2
|
||||
|
||||
from .chunk_delta_h import chunk_gated_delta_rule_fwd_h
|
||||
@@ -81,6 +82,12 @@ def fused_recurrent_kda_fwd(
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
# Bounds-guard instrumentation: the same kernel object as fused_recurrent.py:249, so it
|
||||
# takes the same guard counter. Same site: the defect being guarded is
|
||||
# the kernel's, not the caller's.
|
||||
guard_ctr=guard_telemetry.counter(
|
||||
guard_telemetry.SITE_GDN_RECURRENT_STATE, q.device
|
||||
),
|
||||
scale=scale,
|
||||
N=N,
|
||||
T=T,
|
||||
|
||||
Reference in New Issue
Block a user