[Perf] Fix TRTLLM ragged prefill perf regression (#55499)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
+1
-1
@@ -960,7 +960,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \
|
||||
# Install FlashInfer JIT cache (requires CUDA-version-specific index URL)
|
||||
# https://docs.flashinfer.ai/installation.html
|
||||
# From versions.json: .flashinfer.version
|
||||
ARG FLASHINFER_VERSION=0.6.18
|
||||
ARG FLASHINFER_VERSION=0.6.18.post1
|
||||
RUN --mount=type=cache,target=/opt/uv/cache \
|
||||
if [ "${INSTALL_RUBIN_PRERELEASE}" != "true" ]; then \
|
||||
uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
|
||||
|
||||
@@ -71,7 +71,7 @@
|
||||
"default": "true"
|
||||
},
|
||||
"FLASHINFER_VERSION": {
|
||||
"default": "0.6.18"
|
||||
"default": "0.6.18.post1"
|
||||
},
|
||||
"GDRCOPY_CUDA_VERSION": {
|
||||
"default": "12.8"
|
||||
|
||||
@@ -14,8 +14,8 @@ PyNvVideoCodec==2.0.4
|
||||
# flashinfer-cubin is not on PyPI since 0.6.14; setup.py excludes it from
|
||||
# install_requires so the published wheel does not carry an unresolvable pin
|
||||
--extra-index-url https://flashinfer.ai/whl/
|
||||
flashinfer-python==0.6.18
|
||||
flashinfer-cubin==0.6.18
|
||||
flashinfer-python==0.6.18.post1
|
||||
flashinfer-cubin==0.6.18.post1
|
||||
apache-tvm-ffi==0.1.11
|
||||
tilelang==0.1.12
|
||||
nvidia-cudnn-frontend>=1.19.1
|
||||
|
||||
@@ -57,6 +57,9 @@ def build_chunked_context(
|
||||
([1024, 1024, 1024], 1024),
|
||||
# Unaligned tails.
|
||||
([37, 1000, 5], 1024),
|
||||
# A zero-context request between active requests must not become an
|
||||
# empty TRTLLM ragged-attention row.
|
||||
([4, 0, 2], 1024),
|
||||
([1], 1024),
|
||||
],
|
||||
)
|
||||
@@ -74,6 +77,7 @@ def test_chunks_gather_every_context_row_exactly_once(context_lens, workspace_si
|
||||
gathered: dict[int, list[tuple[int, int]]] = {}
|
||||
previous_request_start = -1
|
||||
for chunk in metadata.chunks:
|
||||
assert chunk.all_rows_active
|
||||
assert chunk.num_context_tokens <= workspace_size
|
||||
assert chunk.num_context_tokens == int(chunk.cu_seq_lens[-1])
|
||||
assert chunk.token_to_seq.shape[0] == chunk.num_context_tokens
|
||||
|
||||
@@ -1547,6 +1547,7 @@ class MLACommonPrefillMetadata:
|
||||
max_seq_len: int
|
||||
seq_lens: torch.Tensor
|
||||
token_to_seq: torch.Tensor
|
||||
all_rows_active: bool
|
||||
|
||||
# for mla DCP
|
||||
padded_local_seq_lens: list[int] | None = None
|
||||
@@ -2124,6 +2125,8 @@ def build_mla_chunked_context_metadata(
|
||||
max_seq_len=max(plan.seq_lens),
|
||||
seq_lens=seq_lens_cpu[request_slice],
|
||||
token_to_seq=token_to_seq[token_slice],
|
||||
all_rows_active=all(length > 0 for length in query_lens)
|
||||
and all(length > 0 for length in plan.seq_lens),
|
||||
num_local_context_tokens=local_token_slice.stop - local_token_slice.start,
|
||||
)
|
||||
if use_dcp:
|
||||
@@ -2529,6 +2532,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
output_dtype=self.model_config.dtype,
|
||||
q_data_type=self.q_data_type,
|
||||
prefill_backend=self._prefill_backend,
|
||||
query_lens_cpu=prefill_query_lens_cpu,
|
||||
)
|
||||
|
||||
self._prefill_backend.prepare_metadata(prefill_metadata)
|
||||
|
||||
@@ -91,6 +91,26 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
self._query_seq_lens = (
|
||||
prefill_metadata.query_start_loc[1:] - prefill_metadata.query_start_loc[:-1]
|
||||
)
|
||||
query_lens_cpu = prefill_metadata.query_lens_cpu
|
||||
if query_lens_cpu is None:
|
||||
raise ValueError("TRTLLM ragged prefill requires CPU query lengths")
|
||||
if query_lens_cpu.device.type != "cpu":
|
||||
raise ValueError("TRTLLM ragged prefill query lengths must be on CPU")
|
||||
if query_lens_cpu.ndim != 1 or query_lens_cpu.numel() == 0:
|
||||
raise ValueError(
|
||||
"TRTLLM ragged prefill requires a non-empty 1D query-length tensor"
|
||||
)
|
||||
min_query_len = int(torch.min(query_lens_cpu).item())
|
||||
if min_query_len < 0:
|
||||
raise ValueError("TRTLLM ragged prefill query lengths must be non-negative")
|
||||
self._has_active_rows = min_query_len > 0
|
||||
if not self._has_active_rows:
|
||||
has_mixed_rows = bool(torch.any(query_lens_cpu > 0).item())
|
||||
if has_mixed_rows:
|
||||
raise ValueError(
|
||||
"TRTLLM ragged prefill contains mixed active and empty query "
|
||||
f"rows: query_lens={query_lens_cpu.tolist()}"
|
||||
)
|
||||
|
||||
def supports_out(self) -> bool:
|
||||
# Output head dim is v.shape[-1] == v_head_dim, so `out` is unpadded.
|
||||
@@ -116,6 +136,18 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
dtype=self._prefill_metadata.output_dtype,
|
||||
)
|
||||
|
||||
if not self._has_active_rows:
|
||||
out.zero_()
|
||||
if return_softmax_lse:
|
||||
lse = torch.empty(
|
||||
self.num_heads,
|
||||
q.shape[0],
|
||||
dtype=torch.float32,
|
||||
device=q.device,
|
||||
)
|
||||
return out, lse.fill_(-float("inf"))
|
||||
return out
|
||||
|
||||
ret = trtllm_ragged_attention_deepseek(
|
||||
query=q,
|
||||
key=k,
|
||||
@@ -135,6 +167,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
is_causal=True,
|
||||
return_lse=return_softmax_lse,
|
||||
out=out,
|
||||
skip_all_rows_active_check=True,
|
||||
)
|
||||
|
||||
if isinstance(ret, tuple):
|
||||
@@ -152,6 +185,11 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from flashinfer.prefill import trtllm_ragged_attention_deepseek
|
||||
|
||||
if not chunk.all_rows_active:
|
||||
raise ValueError(
|
||||
"TRTLLM ragged context prefill contains an empty query or KV row"
|
||||
)
|
||||
|
||||
if out is None:
|
||||
out = torch.empty(
|
||||
q.shape[0],
|
||||
@@ -180,6 +218,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
is_causal=False,
|
||||
return_lse=True,
|
||||
out=out,
|
||||
skip_all_rows_active_check=True,
|
||||
)
|
||||
|
||||
# Convert from (q_len, num_heads) to (num_heads, q_len)
|
||||
|
||||
Reference in New Issue
Block a user