From 26831949b48a0d81fba379dcaf7e378206fd9087 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Wed, 8 Jul 2026 17:59:50 -0500 Subject: [PATCH] [ROCm] Fix pooling startup workspace lock (#47912) Signed-off-by: Andreas Karatzas --- vllm/v1/attention/backends/rocm_attn.py | 3 +++ vllm/v1/attention/backends/triton_attn.py | 1 + .../attention/ops/triton_prefill_attention.py | 16 ++++++++++-- vllm/v1/worker/gpu_worker.py | 25 ++++++++++++++++--- 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/vllm/v1/attention/backends/rocm_attn.py b/vllm/v1/attention/backends/rocm_attn.py index aa81e44e63..24f65ab31a 100644 --- a/vllm/v1/attention/backends/rocm_attn.py +++ b/vllm/v1/attention/backends/rocm_attn.py @@ -288,6 +288,8 @@ class RocmAttentionImpl(AttentionImpl): self.alibi_slopes = alibi_slopes if sliding_window is None: self.sliding_window = (-1, -1) + elif attn_type in (AttentionType.ENCODER, AttentionType.ENCODER_ONLY): + self.sliding_window = (sliding_window - 1, sliding_window - 1) else: self.sliding_window = (sliding_window - 1, 0) self.kv_cache_dtype = kv_cache_dtype @@ -354,6 +356,7 @@ class RocmAttentionImpl(AttentionImpl): softmax_scale=self.scale, sliding_window_q=self.sliding_window[0], sliding_window_k=self.sliding_window[1], + sinks=self.sinks, ) return output diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index 6ba0c94537..7b4a652939 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -760,6 +760,7 @@ class TritonAttentionImpl(AttentionImpl): softmax_scale=self.scale, sliding_window_q=self.sliding_window[0], sliding_window_k=self.sliding_window[1], + sinks=self.sinks, ) return output diff --git a/vllm/v1/attention/ops/triton_prefill_attention.py b/vllm/v1/attention/ops/triton_prefill_attention.py index e9b123fa0d..aaf23c8f4a 100644 --- a/vllm/v1/attention/ops/triton_prefill_attention.py +++ b/vllm/v1/attention/ops/triton_prefill_attention.py @@ -38,6 +38,7 @@ def _fwd_kernel( Q, K, V, + Sinks, sm_scale, B_Start_Loc, B_Seqlen, @@ -57,6 +58,7 @@ def _fwd_kernel( IS_CAUSAL: tl.constexpr, SLIDING_WINDOW_Q: tl.constexpr, SLIDING_WINDOW_K: tl.constexpr, + USE_SINKS: tl.constexpr, Lk: tl.constexpr, ): cur_batch = tl.program_id(0) @@ -94,8 +96,13 @@ def _fwd_kernel( v_ptrs = V + off_v # initialize pointer to m and l - m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") - l_i = tl.zeros([BLOCK_M], dtype=tl.float32) + if USE_SINKS: + sink = tl.load(Sinks + cur_head) * 1.4426950408889634 + m_i = tl.full([BLOCK_M], sink, dtype=tl.float32) + l_i = tl.full([BLOCK_M], 1.0, dtype=tl.float32) + else: + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32) block_mask = tl.where(block_start_loc < cur_batch_seq_len, 1, 0) @@ -200,6 +207,7 @@ def context_attention_fwd( softmax_scale: float | None = None, sliding_window_q: int | None = None, sliding_window_k: int | None = None, + sinks: torch.Tensor | None = None, ): """ q, k, v: [b * s, head, head_dim] @@ -217,6 +225,8 @@ def context_attention_fwd( batch, head = b_seq_len.shape[0], q.shape[1] kv_group_num = q.shape[1] // k.shape[1] + if sinks is not None: + assert sinks.shape[0] == head, "Sinks must be num_query_heads size" grid = (batch, head, triton.cdiv(max_input_len, BLOCK)) num_warps = 4 if Lk <= 64 else 8 @@ -228,6 +238,7 @@ def context_attention_fwd( q, k, v, + sinks if sinks is not None else q, sm_scale, b_start_loc, b_seq_len, @@ -247,6 +258,7 @@ def context_attention_fwd( IS_CAUSAL=is_causal, SLIDING_WINDOW_Q=sliding_window_q, SLIDING_WINDOW_K=sliding_window_k, + USE_SINKS=sinks is not None, num_warps=num_warps, num_stages=1, Lk=Lk, diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 182476e253..327ba6195f 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -729,6 +729,11 @@ class Worker(WorkerBase): @instrument(span_name="Warmup (GPU)") def compile_or_warm_up_model(self) -> CompilationTimes: warmup_sizes: list[int] = [] + cg_capture_sizes: list[int] = [] + + if self.vllm_config.compilation_config.cudagraph_mode != CUDAGraphMode.NONE: + cg_sizes = self.vllm_config.compilation_config.cudagraph_capture_sizes + cg_capture_sizes = [] if cg_sizes is None else cg_sizes if self.vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE: # warm up sizes that are not in cudagraph capture sizes, @@ -736,11 +741,8 @@ class Worker(WorkerBase): # e.g. for the max-num-batched token size in chunked prefill. compile_sizes = self.vllm_config.compilation_config.compile_sizes warmup_sizes = compile_sizes.copy() if compile_sizes is not None else [] # type: ignore[assignment] - cg_capture_sizes: list[int] = [] if self.vllm_config.compilation_config.cudagraph_mode != CUDAGraphMode.NONE: - cg_sizes = self.vllm_config.compilation_config.cudagraph_capture_sizes - cg_capture_sizes = [] if cg_sizes is None else cg_sizes warmup_sizes = [x for x in warmup_sizes if x not in cg_capture_sizes] compile_ranges = self.vllm_config.compilation_config.get_compile_ranges() @@ -753,6 +755,23 @@ class Worker(WorkerBase): if not any(x in compile_range for x in all_sizes): warmup_sizes.append(compile_range.end) + # TODO(LucasWilkinson, akaratza): Remove when MRV1 is deprecated + if ( + current_platform.is_rocm() + and not self.use_v2_model_runner + and self.vllm_config.compilation_config.cudagraph_mode != CUDAGraphMode.NONE + and get_pp_group().is_last_rank + ): + max_num_reqs = min( + self.scheduler_config.max_num_seqs, + self.scheduler_config.max_num_batched_tokens, + ) + if ( + max_num_reqs not in cg_capture_sizes + and max_num_reqs not in warmup_sizes + ): + warmup_sizes.append(max_num_reqs) + # We skip EPLB here since we don't want to record dummy metrics for size in sorted(warmup_sizes, reverse=True): logger.info("Compile and warming up model for size %d", size)