[ROCm] Fix pooling startup workspace lock (#47912)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-07-08 17:59:50 -05:00
committed by GitHub
parent 6cf7b26bd4
commit 26831949b4
4 changed files with 40 additions and 5 deletions
+3
View File
@@ -288,6 +288,8 @@ class RocmAttentionImpl(AttentionImpl):
self.alibi_slopes = alibi_slopes
if sliding_window is None:
self.sliding_window = (-1, -1)
elif attn_type in (AttentionType.ENCODER, AttentionType.ENCODER_ONLY):
self.sliding_window = (sliding_window - 1, sliding_window - 1)
else:
self.sliding_window = (sliding_window - 1, 0)
self.kv_cache_dtype = kv_cache_dtype
@@ -354,6 +356,7 @@ class RocmAttentionImpl(AttentionImpl):
softmax_scale=self.scale,
sliding_window_q=self.sliding_window[0],
sliding_window_k=self.sliding_window[1],
sinks=self.sinks,
)
return output
@@ -760,6 +760,7 @@ class TritonAttentionImpl(AttentionImpl):
softmax_scale=self.scale,
sliding_window_q=self.sliding_window[0],
sliding_window_k=self.sliding_window[1],
sinks=self.sinks,
)
return output
@@ -38,6 +38,7 @@ def _fwd_kernel(
Q,
K,
V,
Sinks,
sm_scale,
B_Start_Loc,
B_Seqlen,
@@ -57,6 +58,7 @@ def _fwd_kernel(
IS_CAUSAL: tl.constexpr,
SLIDING_WINDOW_Q: tl.constexpr,
SLIDING_WINDOW_K: tl.constexpr,
USE_SINKS: tl.constexpr,
Lk: tl.constexpr,
):
cur_batch = tl.program_id(0)
@@ -94,8 +96,13 @@ def _fwd_kernel(
v_ptrs = V + off_v
# initialize pointer to m and l
m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
if USE_SINKS:
sink = tl.load(Sinks + cur_head) * 1.4426950408889634
m_i = tl.full([BLOCK_M], sink, dtype=tl.float32)
l_i = tl.full([BLOCK_M], 1.0, dtype=tl.float32)
else:
m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
block_mask = tl.where(block_start_loc < cur_batch_seq_len, 1, 0)
@@ -200,6 +207,7 @@ def context_attention_fwd(
softmax_scale: float | None = None,
sliding_window_q: int | None = None,
sliding_window_k: int | None = None,
sinks: torch.Tensor | None = None,
):
"""
q, k, v: [b * s, head, head_dim]
@@ -217,6 +225,8 @@ def context_attention_fwd(
batch, head = b_seq_len.shape[0], q.shape[1]
kv_group_num = q.shape[1] // k.shape[1]
if sinks is not None:
assert sinks.shape[0] == head, "Sinks must be num_query_heads size"
grid = (batch, head, triton.cdiv(max_input_len, BLOCK))
num_warps = 4 if Lk <= 64 else 8
@@ -228,6 +238,7 @@ def context_attention_fwd(
q,
k,
v,
sinks if sinks is not None else q,
sm_scale,
b_start_loc,
b_seq_len,
@@ -247,6 +258,7 @@ def context_attention_fwd(
IS_CAUSAL=is_causal,
SLIDING_WINDOW_Q=sliding_window_q,
SLIDING_WINDOW_K=sliding_window_k,
USE_SINKS=sinks is not None,
num_warps=num_warps,
num_stages=1,
Lk=Lk,
+22 -3
View File
@@ -729,6 +729,11 @@ class Worker(WorkerBase):
@instrument(span_name="Warmup (GPU)")
def compile_or_warm_up_model(self) -> CompilationTimes:
warmup_sizes: list[int] = []
cg_capture_sizes: list[int] = []
if self.vllm_config.compilation_config.cudagraph_mode != CUDAGraphMode.NONE:
cg_sizes = self.vllm_config.compilation_config.cudagraph_capture_sizes
cg_capture_sizes = [] if cg_sizes is None else cg_sizes
if self.vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE:
# warm up sizes that are not in cudagraph capture sizes,
@@ -736,11 +741,8 @@ class Worker(WorkerBase):
# e.g. for the max-num-batched token size in chunked prefill.
compile_sizes = self.vllm_config.compilation_config.compile_sizes
warmup_sizes = compile_sizes.copy() if compile_sizes is not None else [] # type: ignore[assignment]
cg_capture_sizes: list[int] = []
if self.vllm_config.compilation_config.cudagraph_mode != CUDAGraphMode.NONE:
cg_sizes = self.vllm_config.compilation_config.cudagraph_capture_sizes
cg_capture_sizes = [] if cg_sizes is None else cg_sizes
warmup_sizes = [x for x in warmup_sizes if x not in cg_capture_sizes]
compile_ranges = self.vllm_config.compilation_config.get_compile_ranges()
@@ -753,6 +755,23 @@ class Worker(WorkerBase):
if not any(x in compile_range for x in all_sizes):
warmup_sizes.append(compile_range.end)
# TODO(LucasWilkinson, akaratza): Remove when MRV1 is deprecated
if (
current_platform.is_rocm()
and not self.use_v2_model_runner
and self.vllm_config.compilation_config.cudagraph_mode != CUDAGraphMode.NONE
and get_pp_group().is_last_rank
):
max_num_reqs = min(
self.scheduler_config.max_num_seqs,
self.scheduler_config.max_num_batched_tokens,
)
if (
max_num_reqs not in cg_capture_sizes
and max_num_reqs not in warmup_sizes
):
warmup_sizes.append(max_num_reqs)
# We skip EPLB here since we don't want to record dummy metrics
for size in sorted(warmup_sizes, reverse=True):
logger.info("Compile and warming up model for size %d", size)