[ROCm] Fix pooling startup workspace lock (#47912)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -288,6 +288,8 @@ class RocmAttentionImpl(AttentionImpl):
|
||||
self.alibi_slopes = alibi_slopes
|
||||
if sliding_window is None:
|
||||
self.sliding_window = (-1, -1)
|
||||
elif attn_type in (AttentionType.ENCODER, AttentionType.ENCODER_ONLY):
|
||||
self.sliding_window = (sliding_window - 1, sliding_window - 1)
|
||||
else:
|
||||
self.sliding_window = (sliding_window - 1, 0)
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
@@ -354,6 +356,7 @@ class RocmAttentionImpl(AttentionImpl):
|
||||
softmax_scale=self.scale,
|
||||
sliding_window_q=self.sliding_window[0],
|
||||
sliding_window_k=self.sliding_window[1],
|
||||
sinks=self.sinks,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@@ -760,6 +760,7 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
softmax_scale=self.scale,
|
||||
sliding_window_q=self.sliding_window[0],
|
||||
sliding_window_k=self.sliding_window[1],
|
||||
sinks=self.sinks,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ def _fwd_kernel(
|
||||
Q,
|
||||
K,
|
||||
V,
|
||||
Sinks,
|
||||
sm_scale,
|
||||
B_Start_Loc,
|
||||
B_Seqlen,
|
||||
@@ -57,6 +58,7 @@ def _fwd_kernel(
|
||||
IS_CAUSAL: tl.constexpr,
|
||||
SLIDING_WINDOW_Q: tl.constexpr,
|
||||
SLIDING_WINDOW_K: tl.constexpr,
|
||||
USE_SINKS: tl.constexpr,
|
||||
Lk: tl.constexpr,
|
||||
):
|
||||
cur_batch = tl.program_id(0)
|
||||
@@ -94,8 +96,13 @@ def _fwd_kernel(
|
||||
v_ptrs = V + off_v
|
||||
|
||||
# initialize pointer to m and l
|
||||
m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
|
||||
l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
|
||||
if USE_SINKS:
|
||||
sink = tl.load(Sinks + cur_head) * 1.4426950408889634
|
||||
m_i = tl.full([BLOCK_M], sink, dtype=tl.float32)
|
||||
l_i = tl.full([BLOCK_M], 1.0, dtype=tl.float32)
|
||||
else:
|
||||
m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
|
||||
l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
|
||||
acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
|
||||
|
||||
block_mask = tl.where(block_start_loc < cur_batch_seq_len, 1, 0)
|
||||
@@ -200,6 +207,7 @@ def context_attention_fwd(
|
||||
softmax_scale: float | None = None,
|
||||
sliding_window_q: int | None = None,
|
||||
sliding_window_k: int | None = None,
|
||||
sinks: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
q, k, v: [b * s, head, head_dim]
|
||||
@@ -217,6 +225,8 @@ def context_attention_fwd(
|
||||
|
||||
batch, head = b_seq_len.shape[0], q.shape[1]
|
||||
kv_group_num = q.shape[1] // k.shape[1]
|
||||
if sinks is not None:
|
||||
assert sinks.shape[0] == head, "Sinks must be num_query_heads size"
|
||||
|
||||
grid = (batch, head, triton.cdiv(max_input_len, BLOCK))
|
||||
num_warps = 4 if Lk <= 64 else 8
|
||||
@@ -228,6 +238,7 @@ def context_attention_fwd(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
sinks if sinks is not None else q,
|
||||
sm_scale,
|
||||
b_start_loc,
|
||||
b_seq_len,
|
||||
@@ -247,6 +258,7 @@ def context_attention_fwd(
|
||||
IS_CAUSAL=is_causal,
|
||||
SLIDING_WINDOW_Q=sliding_window_q,
|
||||
SLIDING_WINDOW_K=sliding_window_k,
|
||||
USE_SINKS=sinks is not None,
|
||||
num_warps=num_warps,
|
||||
num_stages=1,
|
||||
Lk=Lk,
|
||||
|
||||
@@ -729,6 +729,11 @@ class Worker(WorkerBase):
|
||||
@instrument(span_name="Warmup (GPU)")
|
||||
def compile_or_warm_up_model(self) -> CompilationTimes:
|
||||
warmup_sizes: list[int] = []
|
||||
cg_capture_sizes: list[int] = []
|
||||
|
||||
if self.vllm_config.compilation_config.cudagraph_mode != CUDAGraphMode.NONE:
|
||||
cg_sizes = self.vllm_config.compilation_config.cudagraph_capture_sizes
|
||||
cg_capture_sizes = [] if cg_sizes is None else cg_sizes
|
||||
|
||||
if self.vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE:
|
||||
# warm up sizes that are not in cudagraph capture sizes,
|
||||
@@ -736,11 +741,8 @@ class Worker(WorkerBase):
|
||||
# e.g. for the max-num-batched token size in chunked prefill.
|
||||
compile_sizes = self.vllm_config.compilation_config.compile_sizes
|
||||
warmup_sizes = compile_sizes.copy() if compile_sizes is not None else [] # type: ignore[assignment]
|
||||
cg_capture_sizes: list[int] = []
|
||||
|
||||
if self.vllm_config.compilation_config.cudagraph_mode != CUDAGraphMode.NONE:
|
||||
cg_sizes = self.vllm_config.compilation_config.cudagraph_capture_sizes
|
||||
cg_capture_sizes = [] if cg_sizes is None else cg_sizes
|
||||
warmup_sizes = [x for x in warmup_sizes if x not in cg_capture_sizes]
|
||||
|
||||
compile_ranges = self.vllm_config.compilation_config.get_compile_ranges()
|
||||
@@ -753,6 +755,23 @@ class Worker(WorkerBase):
|
||||
if not any(x in compile_range for x in all_sizes):
|
||||
warmup_sizes.append(compile_range.end)
|
||||
|
||||
# TODO(LucasWilkinson, akaratza): Remove when MRV1 is deprecated
|
||||
if (
|
||||
current_platform.is_rocm()
|
||||
and not self.use_v2_model_runner
|
||||
and self.vllm_config.compilation_config.cudagraph_mode != CUDAGraphMode.NONE
|
||||
and get_pp_group().is_last_rank
|
||||
):
|
||||
max_num_reqs = min(
|
||||
self.scheduler_config.max_num_seqs,
|
||||
self.scheduler_config.max_num_batched_tokens,
|
||||
)
|
||||
if (
|
||||
max_num_reqs not in cg_capture_sizes
|
||||
and max_num_reqs not in warmup_sizes
|
||||
):
|
||||
warmup_sizes.append(max_num_reqs)
|
||||
|
||||
# We skip EPLB here since we don't want to record dummy metrics
|
||||
for size in sorted(warmup_sizes, reverse=True):
|
||||
logger.info("Compile and warming up model for size %d", size)
|
||||
|
||||
Reference in New Issue
Block a user