[Bugfix][Attention] Stabilize sparse-MLA DCP for GLM PCP evals (#55879)
Signed-off-by: khluu <khluu000@gmail.com> Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Jason Yao <wsyjh8@gmail.com>
This commit is contained in:
co-authored by
Codex
Tyler Michael Smith
Jason Yao
parent
836bb3839f
commit
241e9391ed
@@ -3,10 +3,13 @@ accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_concurrency: 100
|
||||
# Preserve KV-cache headroom after CUDA graph memory profiling and bound the
|
||||
# DCP-aware FlashInfer workspace to 2.39 GiB at the 4,096-token scheduler cap.
|
||||
server_args: >-
|
||||
--compilation-config '{"cudagraph_mode":"PIECEWISE"}'
|
||||
--gpu-memory-utilization 0.96
|
||||
--max-model-len 4096
|
||||
--max-num-batched-tokens 32768
|
||||
--max-num-batched-tokens 4096
|
||||
--safetensors-load-strategy prefetch
|
||||
--moe-backend flashinfer_cutlass
|
||||
--prefill-context-parallel-size 4
|
||||
|
||||
@@ -3,8 +3,10 @@ accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_concurrency: 100
|
||||
# Leave headroom for FlashInfer CUTLASS autotuning after KV-cache allocation.
|
||||
server_args: >-
|
||||
--compilation-config '{"cudagraph_mode":"NONE"}'
|
||||
--gpu-memory-utilization 0.85
|
||||
--max-model-len 4096
|
||||
--max-num-batched-tokens 32768
|
||||
--safetensors-load-strategy prefetch
|
||||
|
||||
@@ -13,7 +13,7 @@ from vllm.v1.attention.ops.dcp import (
|
||||
mask_dcp_empty_shards_,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
requires_accelerator = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="needs a CUDA or ROCm device for the Triton pack kernel",
|
||||
)
|
||||
@@ -44,6 +44,7 @@ def _pack(out, lse, world_size, h_per_rank, head_dim, seq_lens, query_start_loc)
|
||||
@pytest.mark.parametrize("world_size,h_per_rank", [(8, 2), (8, 16), (4, 4), (2, 1)])
|
||||
@pytest.mark.parametrize("tokens_per_req", [1, 2, 3])
|
||||
@pytest.mark.parametrize("num_pad_rows", [0, 2])
|
||||
@requires_accelerator
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens_list",
|
||||
[
|
||||
@@ -97,6 +98,7 @@ def test_fused_mask_matches_eager(
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("seed", range(8))
|
||||
@requires_accelerator
|
||||
def test_fused_mask_matches_eager_ragged(dtype, seed):
|
||||
"""Non-uniform per-request query lengths.
|
||||
|
||||
@@ -137,6 +139,7 @@ def test_fused_mask_matches_eager_ragged(dtype, seed):
|
||||
)
|
||||
|
||||
|
||||
@requires_accelerator
|
||||
def test_mask_disabled_is_unmasked():
|
||||
"""Passing no seq_lens/query_start_loc must leave the LSE untouched."""
|
||||
torch.manual_seed(0)
|
||||
@@ -149,3 +152,14 @@ def test_mask_disabled_is_unmasked():
|
||||
|
||||
packed = _pack(out, lse, world_size, h_per_rank, head_dim, None, None)
|
||||
assert torch.isfinite(packed.float()).all()
|
||||
|
||||
|
||||
def test_eager_mask_handles_rank_with_no_local_sequences():
|
||||
"""Padded graph rows are all empty when a DCP rank has no sequences."""
|
||||
lse = torch.randn(8, 4, dtype=torch.float32)
|
||||
seq_lens = torch.empty(0, dtype=torch.int32)
|
||||
query_start_loc = torch.tensor([0], dtype=torch.int32)
|
||||
|
||||
mask_dcp_empty_shards_(lse, seq_lens, query_start_loc)
|
||||
|
||||
assert torch.isneginf(lse).all()
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU-only tests for FlashInfer sparse MLA backend constraints."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
_DEFAULT_WORKSPACE_BUFFER_SIZE,
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
_required_workspace_bytes,
|
||||
compute_trtllm_sparse_mla_workspace_bytes,
|
||||
)
|
||||
|
||||
# Reporter config from vllm#50781: GLM-5.2 (64 q heads), TP=8, DCP=8,
|
||||
# max_num_batched_tokens=16384, FlashInfer 0.6.14.
|
||||
_REPORTER_HEADS_PER_RANK = 64 // 8
|
||||
_REPORTER_DCP = 8
|
||||
_REPORTER_MAX_TOKENS = 16384
|
||||
# From the reported crash: trtllm-gen requested exactly this many bytes for
|
||||
# trtllm_gen_softmax_workspace after an 8 MiB counter carve had left
|
||||
# 404,750,336 of the 413,138,944-byte default buffer.
|
||||
_REPORTER_OBSERVED_SOFTMAX_BYTES = 1_611_661_312
|
||||
_REPORTER_OBSERVED_PRECARVE_BYTES = 8_388_608
|
||||
|
||||
|
||||
def test_reporter_softmax_carve_is_reproduced_byte_exactly():
|
||||
# The crashing step scheduled 12288 tokens (a single 12K-token request);
|
||||
# the softmax slab formula must reproduce the kernel's request exactly:
|
||||
# 8 * (8 heads/rank * 8 dcp) * 12288 tokens * 256 + 1 MiB guard.
|
||||
softmax_bytes = compute_trtllm_sparse_mla_workspace_bytes(
|
||||
base_workspace_bytes=0,
|
||||
dcp_world_size=_REPORTER_DCP,
|
||||
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
|
||||
max_num_batched_tokens=12288,
|
||||
)
|
||||
assert softmax_bytes == _REPORTER_OBSERVED_SOFTMAX_BYTES
|
||||
|
||||
|
||||
def test_reporter_config_covers_observed_overflow():
|
||||
computed = compute_trtllm_sparse_mla_workspace_bytes(
|
||||
base_workspace_bytes=_DEFAULT_WORKSPACE_BUFFER_SIZE,
|
||||
dcp_world_size=_REPORTER_DCP,
|
||||
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
|
||||
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
|
||||
)
|
||||
assert computed >= (
|
||||
_REPORTER_OBSERVED_SOFTMAX_BYTES + _REPORTER_OBSERVED_PRECARVE_BYTES
|
||||
)
|
||||
|
||||
|
||||
def test_non_dcp_size_is_unchanged():
|
||||
computed = compute_trtllm_sparse_mla_workspace_bytes(
|
||||
base_workspace_bytes=_DEFAULT_WORKSPACE_BUFFER_SIZE,
|
||||
dcp_world_size=1,
|
||||
num_heads_per_rank=128,
|
||||
max_num_batched_tokens=65536,
|
||||
)
|
||||
assert computed == _DEFAULT_WORKSPACE_BUFFER_SIZE
|
||||
|
||||
|
||||
def test_combined_pcp_dcp_rejects_flashinfer_sparse():
|
||||
config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=4,
|
||||
decode_context_parallel_size=4,
|
||||
),
|
||||
model_config=None,
|
||||
)
|
||||
with patch("vllm.config.get_current_vllm_config", return_value=config):
|
||||
reason = FlashInferMLASparseTRTLLMBackend.supports_combination(
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="fp8",
|
||||
block_size=64,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=True,
|
||||
use_mm_prefix=False,
|
||||
device_capability=SimpleNamespace(major=10),
|
||||
)
|
||||
|
||||
assert reason is not None
|
||||
assert "use FLASHMLA_SPARSE" in reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pcp_size,dcp_size", [(4, 1), (1, 4)])
|
||||
def test_single_context_parallel_mode_allows_flashinfer_sparse(
|
||||
pcp_size: int, dcp_size: int
|
||||
):
|
||||
config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=pcp_size,
|
||||
decode_context_parallel_size=dcp_size,
|
||||
),
|
||||
model_config=None,
|
||||
)
|
||||
with patch("vllm.config.get_current_vllm_config", return_value=config):
|
||||
reason = FlashInferMLASparseTRTLLMBackend.supports_combination(
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="fp8",
|
||||
block_size=64,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=True,
|
||||
use_mm_prefix=False,
|
||||
device_capability=SimpleNamespace(major=10),
|
||||
)
|
||||
|
||||
assert reason is None
|
||||
|
||||
|
||||
def test_default_constant_matches_envs_default(monkeypatch):
|
||||
from vllm import envs
|
||||
|
||||
monkeypatch.delenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", raising=False)
|
||||
assert _DEFAULT_WORKSPACE_BUFFER_SIZE == envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE
|
||||
|
||||
|
||||
def test_env_unset_returns_computed(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", raising=False)
|
||||
required = _required_workspace_bytes(
|
||||
dcp_world_size=_REPORTER_DCP,
|
||||
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
|
||||
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
|
||||
)
|
||||
assert required == compute_trtllm_sparse_mla_workspace_bytes(
|
||||
_DEFAULT_WORKSPACE_BUFFER_SIZE,
|
||||
_REPORTER_DCP,
|
||||
_REPORTER_HEADS_PER_RANK,
|
||||
_REPORTER_MAX_TOKENS,
|
||||
)
|
||||
|
||||
|
||||
def test_env_override_below_computed_is_respected_with_warning(monkeypatch):
|
||||
override = 100 * 1024 * 1024
|
||||
monkeypatch.setenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", str(override))
|
||||
with patch(
|
||||
"vllm.v1.attention.backends.mla.flashinfer_mla_sparse.logger"
|
||||
) as mock_logger:
|
||||
required = _required_workspace_bytes(
|
||||
dcp_world_size=_REPORTER_DCP,
|
||||
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
|
||||
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
|
||||
)
|
||||
assert required == override
|
||||
mock_logger.warning_once.assert_called_once()
|
||||
|
||||
|
||||
def test_env_override_above_computed_is_respected_without_warning(monkeypatch):
|
||||
override = 8 * 1024 * 1024 * 1024
|
||||
monkeypatch.setenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", str(override))
|
||||
with patch(
|
||||
"vllm.v1.attention.backends.mla.flashinfer_mla_sparse.logger"
|
||||
) as mock_logger:
|
||||
required = _required_workspace_bytes(
|
||||
dcp_world_size=_REPORTER_DCP,
|
||||
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
|
||||
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
|
||||
)
|
||||
assert required == override
|
||||
mock_logger.warning_once.assert_not_called()
|
||||
@@ -307,6 +307,23 @@ def test_get_dcp_local_seq_lens_preserves_mtp_bounds_shape():
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
def test_get_dcp_local_seq_lens_reuses_device_rank_tensor(monkeypatch):
|
||||
seq_lens = torch.tensor([0, 1, 7, 8, 17], dtype=torch.int32)
|
||||
world = 4
|
||||
rank = 2
|
||||
rank_tensor = torch.tensor(rank, dtype=torch.int32)
|
||||
expected = get_dcp_local_seq_lens(seq_lens, world, rank, 2)
|
||||
|
||||
def fail_tensor_construction(*args, **kwargs):
|
||||
raise AssertionError("rank tensor must be reused")
|
||||
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(torch, "tensor", fail_tensor_construction)
|
||||
actual = get_dcp_local_seq_lens(seq_lens, world, rank_tensor, 2)
|
||||
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
def test_get_dcp_local_seq_lens_must_run_after_decode_expansion():
|
||||
world = 2
|
||||
rank = 1
|
||||
|
||||
@@ -179,6 +179,7 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
decode_context_parallel_size=1,
|
||||
prefill_context_parallel_size=1,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
),
|
||||
)
|
||||
@@ -193,6 +194,37 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
|
||||
}
|
||||
|
||||
|
||||
def test_indexer_warmup_includes_pcp_normalized_dcp_key(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.mla.indexer.get_dcp_group",
|
||||
lambda: SimpleNamespace(rank_in_group=2),
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
|
||||
model_config=SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(compress_ratios=[32], index_kpool=32)
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
decode_context_parallel_size=4,
|
||||
prefill_context_parallel_size=4,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
),
|
||||
)
|
||||
|
||||
keys = BuildPrefillChunkMetadataKernel().get_warmup_keys(config)
|
||||
|
||||
# Triton's compile key normalizes generic i32 values to 2 and divisible
|
||||
# i32 values (including zero) to 16.
|
||||
assert {(key.dcp_rank, key.dcp_world) for key in keys} == {(2, 2), (16, 1)}
|
||||
assert {
|
||||
(
|
||||
key.input_variant.is_aligned("uncompressed_seq_lens"),
|
||||
key.input_variant.is_aligned("cu_compressed_seq_lens"),
|
||||
)
|
||||
for key in keys
|
||||
} == {(False, False), (False, True), (True, False), (True, True)}
|
||||
|
||||
|
||||
def test_compressed_slot_mapping_warmup_includes_index_kpool():
|
||||
config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=256),
|
||||
|
||||
@@ -113,6 +113,16 @@ class FlashInferMLASparseTRTLLMBackend(_FlashInferMLASparseBackendBase):
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if (
|
||||
parallel_config.prefill_context_parallel_size > 1
|
||||
and parallel_config.decode_context_parallel_size > 1
|
||||
):
|
||||
return (
|
||||
"FLASHINFER_MLA_SPARSE does not support combined PCP+DCP; "
|
||||
"use FLASHMLA_SPARSE, which gathers each DCP KV shard before "
|
||||
"running the rank-local PCP prefill queries"
|
||||
)
|
||||
if kv_cache_dtype == "fp8_ds_mla":
|
||||
return (
|
||||
"FLASHINFER_MLA_SPARSE SM10 does not support fp8_ds_mla kv-cache dtype"
|
||||
@@ -265,6 +275,31 @@ class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilde
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
|
||||
hisparse_supports_multi_token_decode: ClassVar[bool] = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
layer_names: list[str],
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
|
||||
# Under DCP the workspace must hold the trtllm-gen softmax-stats slab.
|
||||
# The buffer address is baked into CUDA graphs, so it has to reach its
|
||||
# final size here, before the first forward/capture (no lazy regrow).
|
||||
if self.dcp_world_size > 1:
|
||||
num_q_heads = vllm_config.model_config.get_num_attention_heads(
|
||||
vllm_config.parallel_config
|
||||
)
|
||||
_get_workspace_buffer(
|
||||
device,
|
||||
_required_workspace_bytes(
|
||||
self.dcp_world_size,
|
||||
num_q_heads,
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
),
|
||||
)
|
||||
|
||||
def _build_req_id_per_token(
|
||||
self,
|
||||
common_attn_metadata: "CommonAttentionMetadata",
|
||||
@@ -275,14 +310,98 @@ class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilde
|
||||
# Global workspace buffer (lazily initialized)
|
||||
_fi_sparse_workspace: torch.Tensor | None = None
|
||||
|
||||
# trtllm-gen carves a softmax-stats slab from the workspace whenever LSE is
|
||||
# requested (FlashInfer csrc/trtllm_fmha_kernel_launcher.cu, unchanged from
|
||||
# v0.6.14 through current main):
|
||||
# sizeof(float2) * num_qo_heads * batch_size * round_up(max_q_len, 256)
|
||||
# + 1 MiB guard
|
||||
# forward_mqa always passes q_len == 1, so each (head, token) pair costs
|
||||
# round_up(1, 256) == 256 slots.
|
||||
_TRTLLM_GEN_SOFTMAX_STAT_BYTES = 8 # sizeof(float2)
|
||||
_TRTLLM_GEN_SOFTMAX_SLOTS_PER_TOKEN = 256 # round_up(q_len=1, 256)
|
||||
_TRTLLM_GEN_SOFTMAX_GUARD_BYTES = 1024 * 1024
|
||||
|
||||
def _get_workspace_buffer(device: torch.device) -> torch.Tensor:
|
||||
# Keep in sync with the VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE default in
|
||||
# vllm/envs.py.
|
||||
_DEFAULT_WORKSPACE_BUFFER_SIZE = 394 * 1024 * 1024
|
||||
|
||||
|
||||
def compute_trtllm_sparse_mla_workspace_bytes(
|
||||
base_workspace_bytes: int,
|
||||
dcp_world_size: int,
|
||||
num_heads_per_rank: int,
|
||||
max_num_batched_tokens: int,
|
||||
) -> int:
|
||||
"""Workspace bytes needed by the trtllm-gen sparse MLA decode kernel.
|
||||
|
||||
Under DCP the query is all-gathered across the DCP group in the head dim
|
||||
and the kernel is asked for LSE, so trtllm-gen carves the softmax-stats
|
||||
slab described above from the workspace before its (batch-independent)
|
||||
counter and scratch regions. ``base_workspace_bytes`` must stay available
|
||||
for those regions, so the slab is added on top of it (see #50781).
|
||||
|
||||
Without DCP no LSE is requested, no slab is carved, and the base size is
|
||||
returned unchanged.
|
||||
"""
|
||||
if dcp_world_size <= 1:
|
||||
return base_workspace_bytes
|
||||
softmax_bytes = (
|
||||
_TRTLLM_GEN_SOFTMAX_STAT_BYTES
|
||||
* (num_heads_per_rank * dcp_world_size)
|
||||
* max_num_batched_tokens
|
||||
* _TRTLLM_GEN_SOFTMAX_SLOTS_PER_TOKEN
|
||||
+ _TRTLLM_GEN_SOFTMAX_GUARD_BYTES
|
||||
)
|
||||
return base_workspace_bytes + softmax_bytes
|
||||
|
||||
|
||||
def _required_workspace_bytes(
|
||||
dcp_world_size: int,
|
||||
num_heads_per_rank: int,
|
||||
max_num_batched_tokens: int,
|
||||
) -> int:
|
||||
"""Resolve the workspace size, honoring an explicit env override."""
|
||||
computed = compute_trtllm_sparse_mla_workspace_bytes(
|
||||
_DEFAULT_WORKSPACE_BUFFER_SIZE,
|
||||
dcp_world_size,
|
||||
num_heads_per_rank,
|
||||
max_num_batched_tokens,
|
||||
)
|
||||
if not envs.is_set("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE"):
|
||||
return computed
|
||||
env_bytes = envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE
|
||||
if env_bytes < computed:
|
||||
logger.warning_once(
|
||||
"VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE=%d is below the %d bytes "
|
||||
"computed for the sparse MLA workspace with dcp_world_size=%d, "
|
||||
"%d heads/rank and max_num_batched_tokens=%d. Respecting the "
|
||||
"override, but the trtllm-gen kernel may crash with a workspace "
|
||||
"overflow; set VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE=%d or unset "
|
||||
"it to use the computed size.",
|
||||
env_bytes,
|
||||
computed,
|
||||
dcp_world_size,
|
||||
num_heads_per_rank,
|
||||
max_num_batched_tokens,
|
||||
computed,
|
||||
)
|
||||
return env_bytes
|
||||
|
||||
|
||||
def _get_workspace_buffer(
|
||||
device: torch.device, min_bytes: int | None = None
|
||||
) -> torch.Tensor:
|
||||
global _fi_sparse_workspace
|
||||
if _fi_sparse_workspace is None:
|
||||
required = (
|
||||
min_bytes
|
||||
if min_bytes is not None
|
||||
else envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE
|
||||
)
|
||||
if _fi_sparse_workspace is None or _fi_sparse_workspace.numel() < required:
|
||||
# FlashInfer's CuteDSL MLA-decode tactic requires an int8 workspace;
|
||||
# the trtllm-gen path views it as uint8, so int8 is safe for all backends.
|
||||
_fi_sparse_workspace = torch.zeros(
|
||||
envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,
|
||||
required,
|
||||
dtype=torch.int8,
|
||||
device=device,
|
||||
)
|
||||
|
||||
@@ -10,7 +10,7 @@ import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_dcp_group, get_pcp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
|
||||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher, zip_inputs
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonPointerInputVariant,
|
||||
@@ -493,6 +493,14 @@ class BuildPrefillChunkMetadataKernel(
|
||||
dcp_world = parallel_config.decode_context_parallel_size
|
||||
dcp_interleave = parallel_config.cp_kv_cache_interleave_size
|
||||
dcp_rank = get_dcp_group().rank_in_group if dcp_world > 1 else 0
|
||||
dcp_cases = [dict(DCP_RANK=dcp_rank, DCP_WORLD=dcp_world)]
|
||||
if parallel_config.prefill_context_parallel_size > 1 and dcp_world > 1:
|
||||
# PCP+DCP prefill constructs a global PCP chunk plan and therefore
|
||||
# dispatches this kernel with already-localized row starts. The
|
||||
# runtime key is normalized to rank 0/world 1 in that path; warm it
|
||||
# explicitly so one PCP rank cannot enter a collective while
|
||||
# another rank is still compiling the first real inference batch.
|
||||
dcp_cases.append(dict(DCP_RANK=0, DCP_WORLD=1))
|
||||
compress_ratios = tuple(
|
||||
dict.fromkeys(
|
||||
max(1, int(ratio))
|
||||
@@ -506,17 +514,33 @@ class BuildPrefillChunkMetadataKernel(
|
||||
if index_kpool and index_kpool > 1 and index_kpool not in compress_ratios:
|
||||
compress_ratios = compress_ratios + (index_kpool,)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
zip_inputs(*dcp_cases),
|
||||
# Cover Triton's divisible, exact-one, and generic i32 classes.
|
||||
query_slice_start=(0, 1, 2),
|
||||
query_slice_stop=(1, 2 * max_tokens - 1, 2 * max_tokens),
|
||||
DCP_RANK=dcp_rank,
|
||||
DCP_WORLD=dcp_world,
|
||||
DCP_INTERLEAVE=dcp_interleave,
|
||||
BLOCK_SIZE=self.BLOCK_SIZE,
|
||||
COMPRESS_RATIO=list(compress_ratios),
|
||||
# PCP's global cumulative lengths are the second row of one packed
|
||||
# allocation, so their pointer is not always 16-byte aligned.
|
||||
# Prefill chunking can independently unalign uncompressed lengths.
|
||||
input_variant=(
|
||||
TritonPointerInputVariant.from_alignment(uncompressed_seq_lens=True),
|
||||
TritonPointerInputVariant.from_alignment(uncompressed_seq_lens=False),
|
||||
TritonPointerInputVariant.from_alignment(
|
||||
uncompressed_seq_lens=True,
|
||||
cu_compressed_seq_lens=True,
|
||||
),
|
||||
TritonPointerInputVariant.from_alignment(
|
||||
uncompressed_seq_lens=True,
|
||||
cu_compressed_seq_lens=False,
|
||||
),
|
||||
TritonPointerInputVariant.from_alignment(
|
||||
uncompressed_seq_lens=False,
|
||||
cu_compressed_seq_lens=True,
|
||||
),
|
||||
TritonPointerInputVariant.from_alignment(
|
||||
uncompressed_seq_lens=False,
|
||||
cu_compressed_seq_lens=False,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -527,7 +551,9 @@ class BuildPrefillChunkMetadataKernel(
|
||||
uncompressed_seq_lens=compile_key.input_variant.pointer(
|
||||
"uncompressed_seq_lens", torch.int32
|
||||
),
|
||||
cu_compressed_seq_lens=int32_ptr,
|
||||
cu_compressed_seq_lens=compile_key.input_variant.pointer(
|
||||
"cu_compressed_seq_lens", torch.int32
|
||||
),
|
||||
row_start_cu_compressed_seq_lens=int32_ptr,
|
||||
token_to_seq=int32_ptr,
|
||||
cu_compressed_seq_len_ks=int32_ptr,
|
||||
@@ -836,6 +862,12 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
# Materialize the rank on device during builder initialization. Creating
|
||||
# this scalar in build() would introduce a GPU<->CPU sync in the decode
|
||||
# hot path.
|
||||
self.dcp_rank_tensor = torch.tensor(
|
||||
self.dcp_rank, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.expanded_block_table_buffer = torch.zeros(
|
||||
(scheduler_config.max_num_batched_tokens, block_table_width),
|
||||
dtype=torch.int32,
|
||||
@@ -889,7 +921,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
local_seq_lens = get_dcp_local_seq_lens(
|
||||
seq_lens,
|
||||
self.dcp_world_size,
|
||||
self.dcp_rank,
|
||||
self.dcp_rank_tensor,
|
||||
self.cp_kv_cache_interleave_size,
|
||||
)
|
||||
if seq_lens_is_buffer_view:
|
||||
|
||||
@@ -1094,7 +1094,7 @@ def compute_causal_conv1d_metadata(
|
||||
def get_dcp_local_seq_lens(
|
||||
seq_lens: torch.Tensor,
|
||||
dcp_size: int = 1,
|
||||
dcp_rank: int | None = None,
|
||||
dcp_rank: int | torch.Tensor | None = None,
|
||||
cp_kv_cache_interleave_size: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""While using dcp, kv_cache size stored on each rank may be different,
|
||||
@@ -1112,6 +1112,12 @@ def get_dcp_local_seq_lens(
|
||||
dcp_size,
|
||||
)
|
||||
seq_lens_tiled = seq_lens_i32.unsqueeze(-1)
|
||||
elif isinstance(dcp_rank, torch.Tensor):
|
||||
assert dcp_rank.dtype == torch.int32
|
||||
assert dcp_rank.device == seq_lens.device
|
||||
assert dcp_rank.numel() == 1
|
||||
rank_offsets = dcp_rank
|
||||
seq_lens_tiled = seq_lens_i32
|
||||
else:
|
||||
rank_offsets = torch.tensor(dcp_rank, dtype=torch.int32, device=seq_lens.device)
|
||||
seq_lens_tiled = seq_lens_i32
|
||||
|
||||
@@ -77,6 +77,13 @@ def mask_dcp_empty_shards_(
|
||||
return
|
||||
assert seq_lens is not None and query_start_loc is not None
|
||||
|
||||
# A DCP rank can receive no local sequences during CUDA graph warmup even
|
||||
# though the padded LSE buffer still has rows. In that case every row is an
|
||||
# empty shard; avoid indexing the empty seq_lens tensor below.
|
||||
if seq_lens.shape[0] == 0:
|
||||
lse.fill_(float("-inf"))
|
||||
return
|
||||
|
||||
row_indices = torch.arange(
|
||||
lse.shape[0], device=lse.device, dtype=query_start_loc.dtype
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user