[Bugfix][Attention] Stabilize sparse-MLA DCP for GLM PCP evals (#55879)

Signed-off-by: khluu <khluu000@gmail.com>
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Jason Yao <wsyjh8@gmail.com>
This commit is contained in:
Kevin H. Luu
2026-09-15 09:58:30 -07:00
committed by GitHub
co-authored by Codex Tyler Michael Smith Jason Yao
parent 836bb3839f
commit 241e9391ed
10 changed files with 411 additions and 13 deletions
@@ -3,10 +3,13 @@ accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
max_concurrency: 100
# Preserve KV-cache headroom after CUDA graph memory profiling and bound the
# DCP-aware FlashInfer workspace to 2.39 GiB at the 4,096-token scheduler cap.
server_args: >-
--compilation-config '{"cudagraph_mode":"PIECEWISE"}'
--gpu-memory-utilization 0.96
--max-model-len 4096
--max-num-batched-tokens 32768
--max-num-batched-tokens 4096
--safetensors-load-strategy prefetch
--moe-backend flashinfer_cutlass
--prefill-context-parallel-size 4
@@ -3,8 +3,10 @@ accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
max_concurrency: 100
# Leave headroom for FlashInfer CUTLASS autotuning after KV-cache allocation.
server_args: >-
--compilation-config '{"cudagraph_mode":"NONE"}'
--gpu-memory-utilization 0.85
--max-model-len 4096
--max-num-batched-tokens 32768
--safetensors-load-strategy prefetch
+15 -1
View File
@@ -13,7 +13,7 @@ from vllm.v1.attention.ops.dcp import (
mask_dcp_empty_shards_,
)
pytestmark = pytest.mark.skipif(
requires_accelerator = pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="needs a CUDA or ROCm device for the Triton pack kernel",
)
@@ -44,6 +44,7 @@ def _pack(out, lse, world_size, h_per_rank, head_dim, seq_lens, query_start_loc)
@pytest.mark.parametrize("world_size,h_per_rank", [(8, 2), (8, 16), (4, 4), (2, 1)])
@pytest.mark.parametrize("tokens_per_req", [1, 2, 3])
@pytest.mark.parametrize("num_pad_rows", [0, 2])
@requires_accelerator
@pytest.mark.parametrize(
"seq_lens_list",
[
@@ -97,6 +98,7 @@ def test_fused_mask_matches_eager(
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("seed", range(8))
@requires_accelerator
def test_fused_mask_matches_eager_ragged(dtype, seed):
"""Non-uniform per-request query lengths.
@@ -137,6 +139,7 @@ def test_fused_mask_matches_eager_ragged(dtype, seed):
)
@requires_accelerator
def test_mask_disabled_is_unmasked():
"""Passing no seq_lens/query_start_loc must leave the LSE untouched."""
torch.manual_seed(0)
@@ -149,3 +152,14 @@ def test_mask_disabled_is_unmasked():
packed = _pack(out, lse, world_size, h_per_rank, head_dim, None, None)
assert torch.isfinite(packed.float()).all()
def test_eager_mask_handles_rank_with_no_local_sequences():
"""Padded graph rows are all empty when a DCP rank has no sequences."""
lse = torch.randn(8, 4, dtype=torch.float32)
seq_lens = torch.empty(0, dtype=torch.int32)
query_start_loc = torch.tensor([0], dtype=torch.int32)
mask_dcp_empty_shards_(lse, seq_lens, query_start_loc)
assert torch.isneginf(lse).all()
@@ -0,0 +1,166 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU-only tests for FlashInfer sparse MLA backend constraints."""
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
_DEFAULT_WORKSPACE_BUFFER_SIZE,
FlashInferMLASparseTRTLLMBackend,
_required_workspace_bytes,
compute_trtllm_sparse_mla_workspace_bytes,
)
# Reporter config from vllm#50781: GLM-5.2 (64 q heads), TP=8, DCP=8,
# max_num_batched_tokens=16384, FlashInfer 0.6.14.
_REPORTER_HEADS_PER_RANK = 64 // 8
_REPORTER_DCP = 8
_REPORTER_MAX_TOKENS = 16384
# From the reported crash: trtllm-gen requested exactly this many bytes for
# trtllm_gen_softmax_workspace after an 8 MiB counter carve had left
# 404,750,336 of the 413,138,944-byte default buffer.
_REPORTER_OBSERVED_SOFTMAX_BYTES = 1_611_661_312
_REPORTER_OBSERVED_PRECARVE_BYTES = 8_388_608
def test_reporter_softmax_carve_is_reproduced_byte_exactly():
# The crashing step scheduled 12288 tokens (a single 12K-token request);
# the softmax slab formula must reproduce the kernel's request exactly:
# 8 * (8 heads/rank * 8 dcp) * 12288 tokens * 256 + 1 MiB guard.
softmax_bytes = compute_trtllm_sparse_mla_workspace_bytes(
base_workspace_bytes=0,
dcp_world_size=_REPORTER_DCP,
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
max_num_batched_tokens=12288,
)
assert softmax_bytes == _REPORTER_OBSERVED_SOFTMAX_BYTES
def test_reporter_config_covers_observed_overflow():
computed = compute_trtllm_sparse_mla_workspace_bytes(
base_workspace_bytes=_DEFAULT_WORKSPACE_BUFFER_SIZE,
dcp_world_size=_REPORTER_DCP,
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
)
assert computed >= (
_REPORTER_OBSERVED_SOFTMAX_BYTES + _REPORTER_OBSERVED_PRECARVE_BYTES
)
def test_non_dcp_size_is_unchanged():
computed = compute_trtllm_sparse_mla_workspace_bytes(
base_workspace_bytes=_DEFAULT_WORKSPACE_BUFFER_SIZE,
dcp_world_size=1,
num_heads_per_rank=128,
max_num_batched_tokens=65536,
)
assert computed == _DEFAULT_WORKSPACE_BUFFER_SIZE
def test_combined_pcp_dcp_rejects_flashinfer_sparse():
config = SimpleNamespace(
parallel_config=SimpleNamespace(
prefill_context_parallel_size=4,
decode_context_parallel_size=4,
),
model_config=None,
)
with patch("vllm.config.get_current_vllm_config", return_value=config):
reason = FlashInferMLASparseTRTLLMBackend.supports_combination(
head_size=576,
dtype=torch.bfloat16,
kv_cache_dtype="fp8",
block_size=64,
use_mla=True,
has_sink=False,
use_sparse=True,
use_mm_prefix=False,
device_capability=SimpleNamespace(major=10),
)
assert reason is not None
assert "use FLASHMLA_SPARSE" in reason
@pytest.mark.parametrize("pcp_size,dcp_size", [(4, 1), (1, 4)])
def test_single_context_parallel_mode_allows_flashinfer_sparse(
pcp_size: int, dcp_size: int
):
config = SimpleNamespace(
parallel_config=SimpleNamespace(
prefill_context_parallel_size=pcp_size,
decode_context_parallel_size=dcp_size,
),
model_config=None,
)
with patch("vllm.config.get_current_vllm_config", return_value=config):
reason = FlashInferMLASparseTRTLLMBackend.supports_combination(
head_size=576,
dtype=torch.bfloat16,
kv_cache_dtype="fp8",
block_size=64,
use_mla=True,
has_sink=False,
use_sparse=True,
use_mm_prefix=False,
device_capability=SimpleNamespace(major=10),
)
assert reason is None
def test_default_constant_matches_envs_default(monkeypatch):
from vllm import envs
monkeypatch.delenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", raising=False)
assert _DEFAULT_WORKSPACE_BUFFER_SIZE == envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE
def test_env_unset_returns_computed(monkeypatch):
monkeypatch.delenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", raising=False)
required = _required_workspace_bytes(
dcp_world_size=_REPORTER_DCP,
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
)
assert required == compute_trtllm_sparse_mla_workspace_bytes(
_DEFAULT_WORKSPACE_BUFFER_SIZE,
_REPORTER_DCP,
_REPORTER_HEADS_PER_RANK,
_REPORTER_MAX_TOKENS,
)
def test_env_override_below_computed_is_respected_with_warning(monkeypatch):
override = 100 * 1024 * 1024
monkeypatch.setenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", str(override))
with patch(
"vllm.v1.attention.backends.mla.flashinfer_mla_sparse.logger"
) as mock_logger:
required = _required_workspace_bytes(
dcp_world_size=_REPORTER_DCP,
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
)
assert required == override
mock_logger.warning_once.assert_called_once()
def test_env_override_above_computed_is_respected_without_warning(monkeypatch):
override = 8 * 1024 * 1024 * 1024
monkeypatch.setenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", str(override))
with patch(
"vllm.v1.attention.backends.mla.flashinfer_mla_sparse.logger"
) as mock_logger:
required = _required_workspace_bytes(
dcp_world_size=_REPORTER_DCP,
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
)
assert required == override
mock_logger.warning_once.assert_not_called()
@@ -307,6 +307,23 @@ def test_get_dcp_local_seq_lens_preserves_mtp_bounds_shape():
torch.testing.assert_close(actual, expected)
def test_get_dcp_local_seq_lens_reuses_device_rank_tensor(monkeypatch):
seq_lens = torch.tensor([0, 1, 7, 8, 17], dtype=torch.int32)
world = 4
rank = 2
rank_tensor = torch.tensor(rank, dtype=torch.int32)
expected = get_dcp_local_seq_lens(seq_lens, world, rank, 2)
def fail_tensor_construction(*args, **kwargs):
raise AssertionError("rank tensor must be reused")
with monkeypatch.context() as patch:
patch.setattr(torch, "tensor", fail_tensor_construction)
actual = get_dcp_local_seq_lens(seq_lens, world, rank_tensor, 2)
torch.testing.assert_close(actual, expected)
def test_get_dcp_local_seq_lens_must_run_after_decode_expansion():
world = 2
rank = 1
@@ -179,6 +179,7 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
),
parallel_config=SimpleNamespace(
decode_context_parallel_size=1,
prefill_context_parallel_size=1,
cp_kv_cache_interleave_size=1,
),
)
@@ -193,6 +194,37 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
}
def test_indexer_warmup_includes_pcp_normalized_dcp_key(monkeypatch):
monkeypatch.setattr(
"vllm.v1.attention.backends.mla.indexer.get_dcp_group",
lambda: SimpleNamespace(rank_in_group=2),
)
config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
model_config=SimpleNamespace(
hf_text_config=SimpleNamespace(compress_ratios=[32], index_kpool=32)
),
parallel_config=SimpleNamespace(
decode_context_parallel_size=4,
prefill_context_parallel_size=4,
cp_kv_cache_interleave_size=1,
),
)
keys = BuildPrefillChunkMetadataKernel().get_warmup_keys(config)
# Triton's compile key normalizes generic i32 values to 2 and divisible
# i32 values (including zero) to 16.
assert {(key.dcp_rank, key.dcp_world) for key in keys} == {(2, 2), (16, 1)}
assert {
(
key.input_variant.is_aligned("uncompressed_seq_lens"),
key.input_variant.is_aligned("cu_compressed_seq_lens"),
)
for key in keys
} == {(False, False), (False, True), (True, False), (True, True)}
def test_compressed_slot_mapping_warmup_includes_index_kpool():
config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=256),
@@ -113,6 +113,16 @@ class FlashInferMLASparseTRTLLMBackend(_FlashInferMLASparseBackendBase):
from vllm.config import get_current_vllm_config
vllm_config = get_current_vllm_config()
parallel_config = vllm_config.parallel_config
if (
parallel_config.prefill_context_parallel_size > 1
and parallel_config.decode_context_parallel_size > 1
):
return (
"FLASHINFER_MLA_SPARSE does not support combined PCP+DCP; "
"use FLASHMLA_SPARSE, which gathers each DCP KV shard before "
"running the rank-local PCP prefill queries"
)
if kv_cache_dtype == "fp8_ds_mla":
return (
"FLASHINFER_MLA_SPARSE SM10 does not support fp8_ds_mla kv-cache dtype"
@@ -265,6 +275,31 @@ class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilde
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
hisparse_supports_multi_token_decode: ClassVar[bool] = True
def __init__(
self,
kv_cache_spec: AttentionSpec,
layer_names: list[str],
vllm_config: VllmConfig,
device: torch.device,
) -> None:
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
# Under DCP the workspace must hold the trtllm-gen softmax-stats slab.
# The buffer address is baked into CUDA graphs, so it has to reach its
# final size here, before the first forward/capture (no lazy regrow).
if self.dcp_world_size > 1:
num_q_heads = vllm_config.model_config.get_num_attention_heads(
vllm_config.parallel_config
)
_get_workspace_buffer(
device,
_required_workspace_bytes(
self.dcp_world_size,
num_q_heads,
vllm_config.scheduler_config.max_num_batched_tokens,
),
)
def _build_req_id_per_token(
self,
common_attn_metadata: "CommonAttentionMetadata",
@@ -275,14 +310,98 @@ class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilde
# Global workspace buffer (lazily initialized)
_fi_sparse_workspace: torch.Tensor | None = None
# trtllm-gen carves a softmax-stats slab from the workspace whenever LSE is
# requested (FlashInfer csrc/trtllm_fmha_kernel_launcher.cu, unchanged from
# v0.6.14 through current main):
# sizeof(float2) * num_qo_heads * batch_size * round_up(max_q_len, 256)
# + 1 MiB guard
# forward_mqa always passes q_len == 1, so each (head, token) pair costs
# round_up(1, 256) == 256 slots.
_TRTLLM_GEN_SOFTMAX_STAT_BYTES = 8 # sizeof(float2)
_TRTLLM_GEN_SOFTMAX_SLOTS_PER_TOKEN = 256 # round_up(q_len=1, 256)
_TRTLLM_GEN_SOFTMAX_GUARD_BYTES = 1024 * 1024
def _get_workspace_buffer(device: torch.device) -> torch.Tensor:
# Keep in sync with the VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE default in
# vllm/envs.py.
_DEFAULT_WORKSPACE_BUFFER_SIZE = 394 * 1024 * 1024
def compute_trtllm_sparse_mla_workspace_bytes(
base_workspace_bytes: int,
dcp_world_size: int,
num_heads_per_rank: int,
max_num_batched_tokens: int,
) -> int:
"""Workspace bytes needed by the trtllm-gen sparse MLA decode kernel.
Under DCP the query is all-gathered across the DCP group in the head dim
and the kernel is asked for LSE, so trtllm-gen carves the softmax-stats
slab described above from the workspace before its (batch-independent)
counter and scratch regions. ``base_workspace_bytes`` must stay available
for those regions, so the slab is added on top of it (see #50781).
Without DCP no LSE is requested, no slab is carved, and the base size is
returned unchanged.
"""
if dcp_world_size <= 1:
return base_workspace_bytes
softmax_bytes = (
_TRTLLM_GEN_SOFTMAX_STAT_BYTES
* (num_heads_per_rank * dcp_world_size)
* max_num_batched_tokens
* _TRTLLM_GEN_SOFTMAX_SLOTS_PER_TOKEN
+ _TRTLLM_GEN_SOFTMAX_GUARD_BYTES
)
return base_workspace_bytes + softmax_bytes
def _required_workspace_bytes(
dcp_world_size: int,
num_heads_per_rank: int,
max_num_batched_tokens: int,
) -> int:
"""Resolve the workspace size, honoring an explicit env override."""
computed = compute_trtllm_sparse_mla_workspace_bytes(
_DEFAULT_WORKSPACE_BUFFER_SIZE,
dcp_world_size,
num_heads_per_rank,
max_num_batched_tokens,
)
if not envs.is_set("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE"):
return computed
env_bytes = envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE
if env_bytes < computed:
logger.warning_once(
"VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE=%d is below the %d bytes "
"computed for the sparse MLA workspace with dcp_world_size=%d, "
"%d heads/rank and max_num_batched_tokens=%d. Respecting the "
"override, but the trtllm-gen kernel may crash with a workspace "
"overflow; set VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE=%d or unset "
"it to use the computed size.",
env_bytes,
computed,
dcp_world_size,
num_heads_per_rank,
max_num_batched_tokens,
computed,
)
return env_bytes
def _get_workspace_buffer(
device: torch.device, min_bytes: int | None = None
) -> torch.Tensor:
global _fi_sparse_workspace
if _fi_sparse_workspace is None:
required = (
min_bytes
if min_bytes is not None
else envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE
)
if _fi_sparse_workspace is None or _fi_sparse_workspace.numel() < required:
# FlashInfer's CuteDSL MLA-decode tactic requires an int8 workspace;
# the trtllm-gen path views it as uint8, so int8 is safe for all backends.
_fi_sparse_workspace = torch.zeros(
envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,
required,
dtype=torch.int8,
device=device,
)
+39 -7
View File
@@ -10,7 +10,7 @@ import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.distributed import get_dcp_group, get_pcp_group
from vllm.logger import init_logger
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
from vllm.model_executor.warmup.jit_warmup import kernel_launcher, zip_inputs
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonPointerInputVariant,
@@ -493,6 +493,14 @@ class BuildPrefillChunkMetadataKernel(
dcp_world = parallel_config.decode_context_parallel_size
dcp_interleave = parallel_config.cp_kv_cache_interleave_size
dcp_rank = get_dcp_group().rank_in_group if dcp_world > 1 else 0
dcp_cases = [dict(DCP_RANK=dcp_rank, DCP_WORLD=dcp_world)]
if parallel_config.prefill_context_parallel_size > 1 and dcp_world > 1:
# PCP+DCP prefill constructs a global PCP chunk plan and therefore
# dispatches this kernel with already-localized row starts. The
# runtime key is normalized to rank 0/world 1 in that path; warm it
# explicitly so one PCP rank cannot enter a collective while
# another rank is still compiling the first real inference batch.
dcp_cases.append(dict(DCP_RANK=0, DCP_WORLD=1))
compress_ratios = tuple(
dict.fromkeys(
max(1, int(ratio))
@@ -506,17 +514,33 @@ class BuildPrefillChunkMetadataKernel(
if index_kpool and index_kpool > 1 and index_kpool not in compress_ratios:
compress_ratios = compress_ratios + (index_kpool,)
return self._trace_dispatch(self.dispatch)(
zip_inputs(*dcp_cases),
# Cover Triton's divisible, exact-one, and generic i32 classes.
query_slice_start=(0, 1, 2),
query_slice_stop=(1, 2 * max_tokens - 1, 2 * max_tokens),
DCP_RANK=dcp_rank,
DCP_WORLD=dcp_world,
DCP_INTERLEAVE=dcp_interleave,
BLOCK_SIZE=self.BLOCK_SIZE,
COMPRESS_RATIO=list(compress_ratios),
# PCP's global cumulative lengths are the second row of one packed
# allocation, so their pointer is not always 16-byte aligned.
# Prefill chunking can independently unalign uncompressed lengths.
input_variant=(
TritonPointerInputVariant.from_alignment(uncompressed_seq_lens=True),
TritonPointerInputVariant.from_alignment(uncompressed_seq_lens=False),
TritonPointerInputVariant.from_alignment(
uncompressed_seq_lens=True,
cu_compressed_seq_lens=True,
),
TritonPointerInputVariant.from_alignment(
uncompressed_seq_lens=True,
cu_compressed_seq_lens=False,
),
TritonPointerInputVariant.from_alignment(
uncompressed_seq_lens=False,
cu_compressed_seq_lens=True,
),
TritonPointerInputVariant.from_alignment(
uncompressed_seq_lens=False,
cu_compressed_seq_lens=False,
),
),
)
@@ -527,7 +551,9 @@ class BuildPrefillChunkMetadataKernel(
uncompressed_seq_lens=compile_key.input_variant.pointer(
"uncompressed_seq_lens", torch.int32
),
cu_compressed_seq_lens=int32_ptr,
cu_compressed_seq_lens=compile_key.input_variant.pointer(
"cu_compressed_seq_lens", torch.int32
),
row_start_cu_compressed_seq_lens=int32_ptr,
token_to_seq=int32_ptr,
cu_compressed_seq_len_ks=int32_ptr,
@@ -836,6 +862,12 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
dtype=torch.int32,
device=self.device,
)
# Materialize the rank on device during builder initialization. Creating
# this scalar in build() would introduce a GPU<->CPU sync in the decode
# hot path.
self.dcp_rank_tensor = torch.tensor(
self.dcp_rank, dtype=torch.int32, device=self.device
)
self.expanded_block_table_buffer = torch.zeros(
(scheduler_config.max_num_batched_tokens, block_table_width),
dtype=torch.int32,
@@ -889,7 +921,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
local_seq_lens = get_dcp_local_seq_lens(
seq_lens,
self.dcp_world_size,
self.dcp_rank,
self.dcp_rank_tensor,
self.cp_kv_cache_interleave_size,
)
if seq_lens_is_buffer_view:
+7 -1
View File
@@ -1094,7 +1094,7 @@ def compute_causal_conv1d_metadata(
def get_dcp_local_seq_lens(
seq_lens: torch.Tensor,
dcp_size: int = 1,
dcp_rank: int | None = None,
dcp_rank: int | torch.Tensor | None = None,
cp_kv_cache_interleave_size: int = 1,
) -> torch.Tensor:
"""While using dcp, kv_cache size stored on each rank may be different,
@@ -1112,6 +1112,12 @@ def get_dcp_local_seq_lens(
dcp_size,
)
seq_lens_tiled = seq_lens_i32.unsqueeze(-1)
elif isinstance(dcp_rank, torch.Tensor):
assert dcp_rank.dtype == torch.int32
assert dcp_rank.device == seq_lens.device
assert dcp_rank.numel() == 1
rank_offsets = dcp_rank
seq_lens_tiled = seq_lens_i32
else:
rank_offsets = torch.tensor(dcp_rank, dtype=torch.int32, device=seq_lens.device)
seq_lens_tiled = seq_lens_i32
+7
View File
@@ -77,6 +77,13 @@ def mask_dcp_empty_shards_(
return
assert seq_lens is not None and query_start_loc is not None
# A DCP rank can receive no local sequences during CUDA graph warmup even
# though the padded LSE buffer still has rows. In that case every row is an
# empty shard; avoid indexing the empty seq_lens tensor below.
if seq_lens.shape[0] == 0:
lse.fill_(float("-inf"))
return
row_indices = torch.arange(
lse.shape[0], device=lse.device, dtype=query_start_loc.dtype
)