[Feature][PCP] Add DeepSeek-V4.1-Flash support (#59857)
Signed-off-by: wangyicong <wangyicong@bytedance.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Kevin H. Luu
parent
074c83a09a
commit
fa75d3456d
@@ -69,10 +69,11 @@ steps:
|
||||
- tests/models/test_deepseek_v4_vl_rocm.py
|
||||
- tests/models/test_deepseek_v41_decoder_replay_layers.py
|
||||
- tests/models/test_deepseek_v41_replay_batch.py
|
||||
- tests/models/test_deepseek_v41_pcp.py
|
||||
- tests/models/transformers/test_layer_registry_e2e.py
|
||||
- tests/models/test_deepseek_v41_replay_start.py
|
||||
commands:
|
||||
- pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py models/transformers/test_layer_registry_e2e.py models/test_deepseek_v41_replay_start.py models/test_deepseek_v41_decoder_replay_layers.py models/test_deepseek_v41_replay_batch.py
|
||||
- pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py models/transformers/test_layer_registry_e2e.py models/test_deepseek_v41_replay_start.py models/test_deepseek_v41_decoder_replay_layers.py models/test_deepseek_v41_replay_batch.py models/test_deepseek_v41_pcp.py
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Basic Models (Other)"
|
||||
|
||||
@@ -585,6 +585,7 @@ def _check_dummy_hash_model_forward(
|
||||
|
||||
model = model_ops.DeepseekV4Model.__new__(model_ops.DeepseekV4Model)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.parallel_config = vllm_config.parallel_config
|
||||
model.use_native_mega_moe = False
|
||||
model.use_sequence_parallel = False
|
||||
model.fuse_mhc_all_reduce = False
|
||||
|
||||
@@ -14,6 +14,7 @@ These tests cover:
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -157,6 +158,104 @@ def test_dspark_context_kv_matches_query_insert(
|
||||
torch.testing.assert_close(kv, before, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize("kv_mxfp8", [False, True])
|
||||
def test_v41_pcp_swa_cache_matches_non_pcp_writer(kv_mxfp8, monkeypatch):
|
||||
"""PCP SWA writes must preserve the established CUDA cache bytes."""
|
||||
from vllm.models.deepseek_v41 import attention as attention_module
|
||||
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
|
||||
|
||||
torch.manual_seed(43)
|
||||
block_size = 64
|
||||
num_local_tokens = 5
|
||||
num_tokens = 2 * num_local_tokens
|
||||
row_size = 528 if kv_mxfp8 else 584
|
||||
num_blocks = 2
|
||||
page_stride = math.ceil(block_size * row_size / 576) * 576 + 576
|
||||
backing = torch.full(
|
||||
(num_blocks, page_stride), 165, dtype=torch.uint8, device="cuda"
|
||||
)
|
||||
cache = backing.as_strided(
|
||||
(num_blocks, block_size, row_size), (page_stride, row_size, 1)
|
||||
)
|
||||
reference_backing = backing.clone()
|
||||
reference = reference_backing.as_strided(cache.shape, cache.stride())
|
||||
|
||||
local_kv = torch.randn(num_local_tokens, 512, dtype=torch.bfloat16, device="cuda")
|
||||
remote_kv = torch.randn_like(local_kv)
|
||||
gathered_kv = torch.cat((local_kv, remote_kv))
|
||||
local_positions = torch.arange(num_local_tokens, device="cuda") + 3
|
||||
cache_positions = torch.cat((local_positions, local_positions + 11))
|
||||
cache_slots = torch.randperm(
|
||||
num_blocks * block_size, dtype=torch.int64, device="cuda"
|
||||
)[:num_tokens]
|
||||
cache_slots[2] = -1
|
||||
angles = torch.randn(32, 32, device="cuda")
|
||||
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
|
||||
num_heads = 8
|
||||
q = torch.randn(
|
||||
num_local_tokens,
|
||||
num_heads,
|
||||
512,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
torch.zeros(num_tokens, num_heads, 512, dtype=torch.bfloat16, device="cuda"),
|
||||
gathered_kv,
|
||||
reference.view(num_blocks, -1),
|
||||
cache_slots,
|
||||
cache_positions,
|
||||
cos_sin,
|
||||
num_heads,
|
||||
1e-20,
|
||||
block_size,
|
||||
False,
|
||||
kv_mxfp8,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
attention_module,
|
||||
"get_pcp_group",
|
||||
lambda: SimpleNamespace(
|
||||
all_gather=lambda tensor, dim: torch.cat((tensor, remote_kv), dim=dim)
|
||||
),
|
||||
)
|
||||
metadata = SimpleNamespace(
|
||||
slot_mapping=torch.full(
|
||||
(num_local_tokens,), -1, dtype=torch.int64, device="cuda"
|
||||
),
|
||||
cache_slot_mapping=cache_slots,
|
||||
cache_positions=cache_positions,
|
||||
block_size=block_size,
|
||||
)
|
||||
attention = SimpleNamespace(
|
||||
use_pcp=True,
|
||||
swa_cache_layer=SimpleNamespace(
|
||||
prefix="swa", kv_cache=cache, block_size=block_size
|
||||
),
|
||||
rotary_emb=SimpleNamespace(cos_sin_cache=cos_sin),
|
||||
_flashinfer_fp8_kv_scale=None,
|
||||
kv_mxfp8=kv_mxfp8,
|
||||
accepts_unnormed_unroped_query=False,
|
||||
n_local_heads=num_heads,
|
||||
padded_heads=num_heads,
|
||||
eps=1e-20,
|
||||
head_dim=512,
|
||||
)
|
||||
DeepseekV4Attention._fused_qnorm_rope_kv_insert(
|
||||
attention, q, local_kv, local_positions, {"swa": metadata}
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
backing.view(torch.uint8),
|
||||
reference_backing.view(torch.uint8),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="graph capture coverage"
|
||||
)
|
||||
@@ -855,6 +954,7 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
|
||||
capacity = 8
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_batched_tokens = 16
|
||||
vllm_config.parallel_config.prefill_context_parallel_size = 1
|
||||
spec = CircularBufferSpec(
|
||||
block_size=capacity,
|
||||
num_kv_heads=1,
|
||||
@@ -891,6 +991,67 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
|
||||
assert metadata.token_to_req_indices.tolist() == [0, 0, 0, 1, 1, 2, 2]
|
||||
|
||||
|
||||
def test_v41_pcp_compressor_latent_is_gathered_for_cache_writes(monkeypatch):
|
||||
"""PCP compresses locally and publishes the gathered latent to both caches."""
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
from vllm.models.deepseek_v41 import attention as attention_module
|
||||
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
|
||||
|
||||
local_latent = torch.arange(8, dtype=torch.float32).view(2, 4)
|
||||
cache_positions = torch.tensor([0, 1, 10, 11])
|
||||
observed = {}
|
||||
|
||||
class Compressor:
|
||||
def __call__(self, kv_score, positions):
|
||||
return local_latent
|
||||
|
||||
def insert_cache(self, latent, positions, rotary_emb):
|
||||
observed["insert"] = (latent, positions)
|
||||
|
||||
def prepare_indexer(qr, latent, weights, positions, rotary, qr_scale, **kwargs):
|
||||
observed["indexer"] = (latent, kwargs)
|
||||
return None, None, None
|
||||
|
||||
monkeypatch.setattr(
|
||||
attention_module,
|
||||
"get_pcp_group",
|
||||
lambda: SimpleNamespace(
|
||||
all_gather=lambda tensor, dim: torch.cat((tensor, tensor + 10), dim=dim)
|
||||
),
|
||||
)
|
||||
metadata = {"swa": SimpleNamespace(cache_positions=cache_positions)}
|
||||
attention = SimpleNamespace(
|
||||
use_pcp=True,
|
||||
compressor=Compressor(),
|
||||
indexer=prepare_indexer,
|
||||
aux_stream_list=None,
|
||||
ln_events=[None, None],
|
||||
swa_cache_layer=SimpleNamespace(prefix="swa"),
|
||||
rotary_emb=SimpleNamespace(),
|
||||
indexer_rotary_emb=SimpleNamespace(),
|
||||
n_local_heads=1,
|
||||
head_dim=4,
|
||||
_wq_b_proj=lambda qr, scale: qr,
|
||||
_fused_qnorm_rope_kv_insert=lambda q, kv, pos, meta: q,
|
||||
_sparse_indexer_and_attn=lambda *args: None,
|
||||
)
|
||||
rows = torch.zeros(2, 4)
|
||||
positions = torch.arange(2)
|
||||
with override_forward_context(ForwardContext({}, metadata, {})):
|
||||
DeepseekV4Attention._prepare_and_attn(
|
||||
attention, rows, rows, rows, None, rows, rows, positions, rows
|
||||
)
|
||||
|
||||
expected_global = torch.cat((local_latent, local_latent + 10))
|
||||
insert_latent, insert_positions = observed["insert"]
|
||||
torch.testing.assert_close(insert_latent, expected_global)
|
||||
torch.testing.assert_close(insert_positions, cache_positions)
|
||||
indexer_latent, indexer_kwargs = observed["indexer"]
|
||||
torch.testing.assert_close(indexer_latent, local_latent)
|
||||
torch.testing.assert_close(indexer_kwargs["cache_latent"], expected_global)
|
||||
torch.testing.assert_close(indexer_kwargs["cache_positions"], cache_positions)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA stream coverage")
|
||||
@pytest.mark.parametrize(
|
||||
"use_aux,use_graph", [(False, False), (True, False), (True, True)]
|
||||
@@ -930,6 +1091,7 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
|
||||
context = ForwardContext({}, metadata, {})
|
||||
compressor = DeepseekCompressor.__new__(DeepseekCompressor)
|
||||
torch.nn.Module.__init__(compressor)
|
||||
compressor.pcp_size = 1
|
||||
compressor.head_dim, compressor.rope_head_dim, compressor.compress_ratio = (
|
||||
512,
|
||||
64,
|
||||
@@ -955,7 +1117,7 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
|
||||
use_fp4_kv=False,
|
||||
)
|
||||
|
||||
def prepare_indexer(qr, latent, weights, positions, rotary, qr_scale):
|
||||
def prepare_indexer(qr, latent, weights, positions, rotary, qr_scale, **kwargs):
|
||||
DeepseekV4Indexer._produce_k(indexer, latent, positions, rotary)
|
||||
return None, None, None
|
||||
|
||||
@@ -965,6 +1127,7 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
|
||||
|
||||
auxiliary = [torch.cuda.Stream()] if use_aux else None
|
||||
attention = SimpleNamespace(
|
||||
use_pcp=False,
|
||||
compressor=compressor,
|
||||
indexer=prepare_indexer,
|
||||
aux_stream_list=auxiliary,
|
||||
@@ -1022,6 +1185,291 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
|
||||
previous = expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pcp_size", [2, 4])
|
||||
@pytest.mark.parametrize(
|
||||
"lengths,prefilling,starts",
|
||||
[
|
||||
([9, 1], [True, False], [1, 4]),
|
||||
([1, 1], [False, False], [16, 3]),
|
||||
([3, 1], [True, False], [1, 0]),
|
||||
],
|
||||
)
|
||||
def test_v41_pcp_ring_links_match_request_positions(
|
||||
pcp_size, lengths, prefilling, starts, monkeypatch
|
||||
):
|
||||
"""Cross-chunk predecessors and open ring writes must belong to the request."""
|
||||
from vllm.models.deepseek_v41 import compressor as compressor_module
|
||||
from vllm.models.deepseek_v41.compressor import (
|
||||
CompressorMetadata,
|
||||
CompressorMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.worker.gpu import pcp_manager as pcp_manager_module
|
||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||
|
||||
monkeypatch.setattr(
|
||||
pcp_manager_module,
|
||||
"async_tensor_h2d",
|
||||
lambda array, **kwargs: torch.from_numpy(array),
|
||||
)
|
||||
manager = PCPManager(pcp_size, 0, torch.device("cpu"), shard_decode_requests=True)
|
||||
query_start = np.array([0, *np.cumsum(lengths)], dtype=np.int32)
|
||||
segments, counts = manager._build_batch_layout(
|
||||
np.array(lengths), np.array(starts), np.array(prefilling), query_start
|
||||
)
|
||||
width = max(counts)
|
||||
capacity = 8
|
||||
blocks = [1, (1 << 20) + 2]
|
||||
num_rows = min(width, 2 * len(lengths))
|
||||
metadata_by_rank = []
|
||||
positions_by_rank = []
|
||||
gathered_keys = []
|
||||
for local_segments in segments:
|
||||
positions = torch.zeros(width, dtype=torch.int64)
|
||||
slots = torch.full((width,), -1, dtype=torch.int64)
|
||||
local_query_start = [0]
|
||||
keys = [-1] * num_rows
|
||||
for row, segment in enumerate(local_segments):
|
||||
req = segment.global_batch_req_idx
|
||||
begin = starts[req] + segment.global_batch_slice.start - query_start[req]
|
||||
local = segment.rank_local_batch_slice
|
||||
positions[local] = torch.arange(begin, begin + segment.num_tokens)
|
||||
slots[local] = blocks[req] * capacity + positions[local] % capacity
|
||||
local_query_start.append(local.stop)
|
||||
keys[row] = (blocks[req] << 32) | int(positions[local.stop - 1])
|
||||
if not local_segments:
|
||||
local_query_start.append(0)
|
||||
metadata_by_rank.append(
|
||||
CompressorMetadata(
|
||||
slot_mapping=slots,
|
||||
query_start_loc=torch.tensor(local_query_start, dtype=torch.int32),
|
||||
token_to_req_indices=torch.empty(0, dtype=torch.int32),
|
||||
)
|
||||
)
|
||||
positions_by_rank.append(positions)
|
||||
gathered_keys.extend(keys)
|
||||
|
||||
keys = torch.tensor(gathered_keys, dtype=torch.int64)
|
||||
contributed_keys = []
|
||||
|
||||
def all_gather(tensor, dim):
|
||||
contributed_keys.append(tensor.clone())
|
||||
return keys
|
||||
|
||||
monkeypatch.setattr(
|
||||
compressor_module,
|
||||
"get_pcp_group",
|
||||
lambda: SimpleNamespace(all_gather=all_gather),
|
||||
)
|
||||
builder = SimpleNamespace(
|
||||
capacity=capacity,
|
||||
device=torch.device("cpu"),
|
||||
vllm_config=SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=len(lengths))
|
||||
),
|
||||
)
|
||||
expected_rows = {
|
||||
block: max(key for key in gathered_keys if key >> 32 == block)
|
||||
for block in blocks
|
||||
}
|
||||
expected_slots = [
|
||||
block * capacity + (key & 0xFFFFFFFF) % capacity
|
||||
for block, key in expected_rows.items()
|
||||
if key % 2 == 0
|
||||
]
|
||||
for metadata, positions in zip(metadata_by_rank, positions_by_rank):
|
||||
num_reqs = metadata.query_start_loc.numel() - 1
|
||||
CompressorMetadataBuilder._link_pcp_chunks(
|
||||
builder, metadata, num_reqs, positions
|
||||
)
|
||||
assert metadata.prev_row_indices is not None
|
||||
assert metadata.gather_tokens is not None
|
||||
assert metadata.ring_write_slots is not None
|
||||
for row, token in enumerate(metadata.gather_tokens.tolist()):
|
||||
key = int(contributed_keys[-1][row])
|
||||
if key >= 0:
|
||||
assert int(metadata.slot_mapping[token]) // capacity == key >> 32
|
||||
assert int(positions[token]) == key & 0xFFFFFFFF
|
||||
for start, prev in zip(
|
||||
metadata.query_start_loc[:-1].tolist(), metadata.prev_row_indices.tolist()
|
||||
):
|
||||
position = int(positions[start])
|
||||
block = int(metadata.slot_mapping[start]) // capacity
|
||||
wanted = (block << 32) | (position - 1)
|
||||
if position % 2 == 1 and wanted in gathered_keys:
|
||||
assert prev >= 0 and gathered_keys[prev] == wanted
|
||||
else:
|
||||
assert prev == -1
|
||||
writes = metadata.ring_write_slots.tolist()
|
||||
assert sorted(slot for slot in writes if slot >= 0) == sorted(expected_slots)
|
||||
for row, slot in enumerate(writes):
|
||||
if slot >= 0:
|
||||
assert gathered_keys[row] == expected_rows[slot // capacity]
|
||||
assert (gathered_keys[row] & 0xFFFFFFFF) % capacity == slot % capacity
|
||||
torch.testing.assert_close(torch.cat(contributed_keys), keys)
|
||||
|
||||
|
||||
def _build_pcp_ring_metadata(ranks, all_gather, spec, config, device, monkeypatch):
|
||||
from vllm.models.deepseek_v41 import compressor as compressor_module
|
||||
from vllm.models.deepseek_v41.compressor import CompressorMetadataBuilder
|
||||
|
||||
monkeypatch.setattr(
|
||||
compressor_module,
|
||||
"get_pcp_group",
|
||||
lambda: SimpleNamespace(all_gather=all_gather),
|
||||
)
|
||||
for local in ranks:
|
||||
builder = CompressorMetadataBuilder(spec, ["ring"], config, device)
|
||||
local.metadata = builder.build(0, local.common)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Triton kernel")
|
||||
@pytest.mark.parametrize("pcp_size", [2, 4])
|
||||
def test_v41_pcp_compressor_preserves_pairs_and_decode_state(pcp_size, monkeypatch):
|
||||
"""Pairs split across PCP chunks and decode owner changes keep exact latents."""
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
from vllm.models.deepseek_v41 import compressor as compressor_module
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
fused_save_compress_norm,
|
||||
)
|
||||
from vllm.models.deepseek_v41.compressor import DeepseekCompressor
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.kv_cache_interface import CircularBufferSpec
|
||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||
|
||||
torch.manual_seed(42)
|
||||
device = torch.device("cuda")
|
||||
norm = torch.randn(512, dtype=torch.bfloat16, device=device)
|
||||
# Block 0 is reserved as the null block; requests use physical blocks 1/2.
|
||||
reference_state = torch.zeros(3, 8, 1024, device=device)
|
||||
rank_states = [reference_state.clone() for _ in range(pcp_size)]
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=32, max_num_seqs=4),
|
||||
parallel_config=SimpleNamespace(prefill_context_parallel_size=pcp_size),
|
||||
speculative_config=None,
|
||||
)
|
||||
spec = CircularBufferSpec(
|
||||
block_size=8, num_kv_heads=1, head_size=1024, head_size_v=0, dtype=torch.float32
|
||||
)
|
||||
starts = np.zeros(2, dtype=np.int32)
|
||||
for lengths, prefilling in [
|
||||
([7, 3], [True, True]),
|
||||
([9, 1], [True, False]),
|
||||
([1, 1], [False, False]),
|
||||
([1, 1], [False, False]),
|
||||
]:
|
||||
lengths = np.array(lengths, dtype=np.int32)
|
||||
query_start = np.concatenate(([0], np.cumsum(lengths))).astype(np.int32)
|
||||
qsl = torch.from_numpy(query_start).to(device)
|
||||
positions = torch.cat(
|
||||
[torch.arange(s, s + n, device=device) for s, n in zip(starts, lengths)]
|
||||
)
|
||||
req_ids = torch.repeat_interleave(
|
||||
torch.arange(2, device=device, dtype=torch.int32),
|
||||
torch.from_numpy(lengths).to(device),
|
||||
)
|
||||
slots = (req_ids.long() + 1) * 8 + positions % 8
|
||||
raw = torch.randn(sum(lengths), 1024, device=device)
|
||||
expected = torch.empty(sum(lengths), 512, dtype=torch.bfloat16, device=device)
|
||||
fused_save_compress_norm(
|
||||
raw,
|
||||
positions,
|
||||
reference_state,
|
||||
slots,
|
||||
qsl,
|
||||
req_ids,
|
||||
norm,
|
||||
1e-20,
|
||||
2,
|
||||
expected,
|
||||
)
|
||||
|
||||
ranks = []
|
||||
for rank in range(pcp_size):
|
||||
manager = PCPManager(pcp_size, rank, device, shard_decode_requests=True)
|
||||
segments, counts = manager._build_batch_layout(
|
||||
lengths, starts, np.array(prefilling), query_start
|
||||
)
|
||||
width = max(counts)
|
||||
local_indices = manager._padded_gather_idx[
|
||||
rank * width : (rank + 1) * width
|
||||
]
|
||||
local_lengths = [s.num_tokens for s in segments[rank]] or [0]
|
||||
local_qsl = torch.tensor([0, *np.cumsum(local_lengths)], dtype=torch.int32)
|
||||
local_reqs = [s.global_batch_req_idx for s in segments[rank]] or [0]
|
||||
common = CommonAttentionMetadata(
|
||||
query_start_loc=local_qsl.to(device),
|
||||
query_start_loc_cpu=local_qsl,
|
||||
seq_lens=torch.zeros(len(local_reqs), dtype=torch.int32, device=device),
|
||||
num_reqs=len(local_reqs),
|
||||
num_actual_tokens=counts[rank],
|
||||
max_query_len=max(local_lengths),
|
||||
max_seq_len=int(max(starts + lengths)),
|
||||
block_table_tensor=torch.tensor(
|
||||
np.array(local_reqs) + 1, dtype=torch.int32, device=device
|
||||
).unsqueeze(1),
|
||||
slot_mapping=torch.full((width * pcp_size,), -1, device=device),
|
||||
positions=positions[local_indices],
|
||||
)
|
||||
ranks.append(
|
||||
SimpleNamespace(
|
||||
indices=local_indices[: counts[rank]],
|
||||
raw=raw[local_indices],
|
||||
positions=common.positions,
|
||||
common=common,
|
||||
)
|
||||
)
|
||||
|
||||
# Each rank's builder contributes its last-row candidates to a gather.
|
||||
candidates: list[torch.Tensor] = []
|
||||
_build_pcp_ring_metadata(
|
||||
ranks,
|
||||
lambda t, dim, out=candidates: out.append(t.clone()) or t.repeat(pcp_size),
|
||||
spec,
|
||||
config,
|
||||
device,
|
||||
monkeypatch,
|
||||
)
|
||||
_build_pcp_ring_metadata(
|
||||
ranks,
|
||||
lambda t, dim, out=candidates: torch.cat(out),
|
||||
spec,
|
||||
config,
|
||||
device,
|
||||
monkeypatch,
|
||||
)
|
||||
gathered_rows = torch.cat([r.raw[r.metadata.gather_tokens] for r in ranks])
|
||||
monkeypatch.setattr(
|
||||
compressor_module,
|
||||
"get_pcp_group",
|
||||
lambda rows=gathered_rows: SimpleNamespace(all_gather=lambda t, dim: rows),
|
||||
)
|
||||
for rank, local in enumerate(ranks):
|
||||
compressor = SimpleNamespace(
|
||||
pcp_size=pcp_size,
|
||||
head_dim=512,
|
||||
compress_ratio=2,
|
||||
rms_norm_eps=1e-20,
|
||||
norm=SimpleNamespace(weight=norm),
|
||||
state_cache=SimpleNamespace(prefix="ring", kv_cache=rank_states[rank]),
|
||||
)
|
||||
context = ForwardContext({}, {"ring": local.metadata}, {})
|
||||
with override_forward_context(context):
|
||||
actual = DeepseekCompressor.forward(
|
||||
compressor, local.raw, local.positions
|
||||
)
|
||||
assert isinstance(actual, torch.Tensor)
|
||||
valid = (positions[local.indices] + 1) % 2 == 0
|
||||
torch.testing.assert_close(
|
||||
actual[: local.indices.numel()][valid],
|
||||
expected[local.indices][valid],
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
for state in rank_states[1:]:
|
||||
torch.testing.assert_close(state, rank_states[0], rtol=0, atol=0)
|
||||
starts += lengths
|
||||
|
||||
|
||||
def _on_gfx950() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
|
||||
@@ -502,11 +502,12 @@ def test_lookback_window_reproduces_single_instance(runner, capture):
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
|
||||
def test_v2_model_state_gathers_lookback_window():
|
||||
@pytest.mark.parametrize("pcp", [False, True])
|
||||
def test_v2_model_state_gathers_lookback_window(pcp):
|
||||
"""The window is gathered on device from the runner's token history."""
|
||||
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
|
||||
|
||||
depth, max_num_reqs, max_model_len = 3, 4, 16
|
||||
depth, max_num_reqs, max_model_len = 3, 6, 16
|
||||
state = DeepseekV41ModelState.__new__(DeepseekV41ModelState)
|
||||
state.rope_state = None
|
||||
state.supports_mm_inputs = False
|
||||
@@ -527,15 +528,23 @@ def test_v2_model_state_gathers_lookback_window():
|
||||
# Batch rows -> request state rows: a request two decode steps in, a fresh
|
||||
# request, and one at the end of a 5-token prompt.
|
||||
input_batch = SimpleNamespace(
|
||||
idx_mapping=torch.tensor([2, 0, 3], dtype=torch.int64, device="cuda")
|
||||
idx_mapping=torch.tensor(
|
||||
[2, 0, 3, 2] if pcp else [2, 0, 3], dtype=torch.int64, device="cuda"
|
||||
),
|
||||
query_start_loc=torch.arange(5 if pcp else 4, dtype=torch.int32, device="cuda"),
|
||||
positions=torch.tensor([8, 0, 6, 5] if pcp else [8, 0, 6], device="cuda"),
|
||||
)
|
||||
window = state.prepare_inputs(input_batch, req_states)["lookback_token_ids"]
|
||||
assert window.cpu().tolist() == [
|
||||
expected = [
|
||||
[22, 21, 16],
|
||||
[-1, -1, -1],
|
||||
[0, 0, 0],
|
||||
[-1, -1, -1],
|
||||
]
|
||||
if pcp:
|
||||
# The second PCP chunk of request 2 starts at 5, not at its global 8.
|
||||
expected.append([15, 14, 13])
|
||||
expected.extend([[-1] * depth] * (max_num_reqs - len(expected)))
|
||||
assert window.cpu().tolist() == expected
|
||||
# Graph capture runs on the dummy inputs and replays read the same buffer.
|
||||
dummy = state.prepare_dummy_inputs(num_reqs=3, num_tokens=3)["lookback_token_ids"]
|
||||
assert dummy is window and torch.all(dummy == -1)
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v41.nvidia import model as model_module
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pcp_size,use_ep", [(1, False), (2, True), (2, False), (4, False)]
|
||||
)
|
||||
@pytest.mark.parametrize("use_sp", [False, True])
|
||||
def test_pcp_gathers_routing_ids_once_after_local_input_preparation(
|
||||
pcp_size, use_ep, use_sp, monkeypatch
|
||||
):
|
||||
"""Embedding keeps local IDs; all decoder layers receive aligned routing IDs."""
|
||||
gather = Mock(side_effect=lambda ids, dim: torch.cat([ids] * pcp_size, dim=dim))
|
||||
monkeypatch.setattr(
|
||||
model_module, "get_pcp_group", lambda: SimpleNamespace(all_gather=gather)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
model_module,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
monkeypatch.setattr(model_module, "is_forward_context_available", lambda: False)
|
||||
monkeypatch.setattr(model_module, "sp_shard", lambda tensor: tensor[::2])
|
||||
monkeypatch.setattr(
|
||||
model_module, "sp_all_gather", lambda tensor: tensor.repeat_interleave(2, dim=0)
|
||||
)
|
||||
monkeypatch.setattr(model_module, "hc_collapse_triton", lambda hidden, mix: hidden)
|
||||
embed = Mock(side_effect=lambda ids: ids[:, None].float())
|
||||
run_layers = Mock(
|
||||
side_effect=lambda layer_ids, hidden, *args: (hidden, None, None, None, hidden)
|
||||
)
|
||||
model = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=pcp_size, enable_expert_parallel=use_ep
|
||||
),
|
||||
use_native_mega_moe=False,
|
||||
use_sequence_parallel=use_sp,
|
||||
engram_hash=None,
|
||||
embed_input_ids=embed,
|
||||
start_layer=0,
|
||||
decoder_replay_start=2,
|
||||
decoder_replay_layers=None,
|
||||
aux_hidden_state_layers=(),
|
||||
_mtp_hidden_buffer=None,
|
||||
_run_layers=run_layers,
|
||||
_collapse=lambda hidden, *args: hidden,
|
||||
norm=lambda hidden: hidden,
|
||||
)
|
||||
for offset in (0, 10):
|
||||
local_ids = torch.tensor([3, 7, 11, 15]) + offset
|
||||
model_module.DeepseekV4Model.forward(model, local_ids, torch.arange(4), None)
|
||||
assert embed.call_args.args[0] is local_ids
|
||||
routing_ids = local_ids[::2] if use_sp else local_ids
|
||||
needs_gather = pcp_size > 1 and not use_ep
|
||||
if needs_gather:
|
||||
torch.testing.assert_close(gather.call_args.args[0], routing_ids)
|
||||
routing_ids = routing_ids.repeat(pcp_size)
|
||||
torch.testing.assert_close(run_layers.call_args.args[3], routing_ids)
|
||||
assert gather.call_count == 2 * int(needs_gather)
|
||||
@@ -9,6 +9,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.attention import AttentionConfig
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.selector import AttentionSelectorConfig
|
||||
@@ -71,6 +72,7 @@ def test_deepseek_v41_rocm_accepts_legacy_backend_alias(backend_name, expect_war
|
||||
|
||||
config = SimpleNamespace(
|
||||
attention_config=AttentionConfig(backend=backend_name),
|
||||
parallel_config=ParallelConfig(),
|
||||
)
|
||||
|
||||
with patch.object(dsv41_model.logger, "warning_once") as warning_once:
|
||||
|
||||
@@ -140,6 +140,10 @@ class DeepseekV4MoE(DeepseekV4MoEBase):
|
||||
|
||||
def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
"""Pick the ROCm sparse-MLA implementation for DeepSeek V4.1."""
|
||||
if vllm_config.parallel_config.prefill_context_parallel_size > 1:
|
||||
raise NotImplementedError(
|
||||
"DeepSeek-V4.1 PCP currently supports NVIDIA CUDA only."
|
||||
)
|
||||
backend = vllm_config.attention_config.backend
|
||||
if backend is not None and backend not in (
|
||||
AttentionBackendEnum.ROCM_FLASHMLA_SPARSE_DSV4,
|
||||
|
||||
@@ -53,7 +53,7 @@ from vllm.config import (
|
||||
get_current_vllm_config,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.distributed import get_pcp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
@@ -297,6 +297,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.prefix = prefix # Alias for compatibility with compressor
|
||||
self.use_pcp = vllm_config.parallel_config.prefill_context_parallel_size > 1
|
||||
self.hidden_size = config.hidden_size
|
||||
self.n_heads = config.num_attention_heads
|
||||
assert self.n_heads % tp_size == 0
|
||||
@@ -878,7 +879,10 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
index_q_scale: torch.Tensor | None = None
|
||||
index_weights_out: torch.Tensor | None = None
|
||||
latent: torch.Tensor | None = None
|
||||
aux_stream = aux_streams[0] if aux_streams is not None else None
|
||||
# Keep PCP collectives on one stream in the same order on every rank.
|
||||
aux_stream = (
|
||||
aux_streams[0] if aux_streams is not None and not self.use_pcp else None
|
||||
)
|
||||
|
||||
if compressor is not None:
|
||||
# Q projection / KV insertion on the default stream overlaps the
|
||||
@@ -893,6 +897,17 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
else:
|
||||
q = project_query_and_cache_kv()
|
||||
|
||||
cache_latent = latent
|
||||
cache_positions = positions
|
||||
if self.use_pcp and latent is not None:
|
||||
assert isinstance(attn_metadata, dict)
|
||||
swa_metadata = cast(
|
||||
"DeepseekSparseSWAMetadata", attn_metadata[self.swa_cache_layer.prefix]
|
||||
)
|
||||
assert swa_metadata.cache_positions is not None
|
||||
cache_latent = get_pcp_group().all_gather(latent, dim=0)
|
||||
cache_positions = swa_metadata.cache_positions
|
||||
|
||||
def prepare_indexer():
|
||||
if indexer is None:
|
||||
return None, None, None
|
||||
@@ -903,12 +918,16 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
positions,
|
||||
self.indexer_rotary_emb,
|
||||
qr_scale,
|
||||
cache_latent=cache_latent,
|
||||
cache_positions=cache_positions,
|
||||
)
|
||||
|
||||
if compressor is not None:
|
||||
indexer_result, _ = maybe_execute_in_parallel(
|
||||
prepare_indexer,
|
||||
lambda: compressor.insert_cache(latent, positions, self.rotary_emb),
|
||||
lambda: compressor.insert_cache(
|
||||
cache_latent, cache_positions, self.rotary_emb
|
||||
),
|
||||
self.ln_events[0],
|
||||
self.ln_events[1],
|
||||
aux_stream,
|
||||
@@ -1071,6 +1090,25 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
assert positions.dtype == torch.int64
|
||||
cos_sin_cache = self.rotary_emb.cos_sin_cache
|
||||
cache_dtype = swa_kv_cache.dtype
|
||||
slot_mapping = swa_metadata.slot_mapping
|
||||
if self.use_pcp:
|
||||
cache_kv = get_pcp_group().all_gather(kv.contiguous(), dim=0)
|
||||
assert swa_metadata.cache_slot_mapping is not None
|
||||
assert swa_metadata.cache_positions is not None
|
||||
torch.ops._C.fused_deepseek_v4_kv_rope_insert(
|
||||
cache_kv,
|
||||
swa_kv_cache,
|
||||
swa_metadata.cache_slot_mapping,
|
||||
swa_metadata.cache_positions,
|
||||
cos_sin_cache,
|
||||
swa_metadata.block_size,
|
||||
self._flashinfer_fp8_kv_scale
|
||||
if cache_dtype == torch.float8_e4m3fn
|
||||
else None,
|
||||
self.kv_mxfp8,
|
||||
)
|
||||
# The fused op now only prepares the rank-local queries.
|
||||
slot_mapping = slot_mapping[:0]
|
||||
|
||||
# kv is unchanged; attention reads kv solely via swa_kv_cache.
|
||||
if cache_dtype == torch.uint8:
|
||||
@@ -1092,7 +1130,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
q,
|
||||
kv,
|
||||
swa_kv_cache_2d,
|
||||
swa_metadata.slot_mapping,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
pad_to,
|
||||
@@ -1120,7 +1158,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
q,
|
||||
kv,
|
||||
swa_kv_cache_3d,
|
||||
swa_metadata.slot_mapping,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
self.eps,
|
||||
@@ -1136,7 +1174,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
kv,
|
||||
q_fp8,
|
||||
swa_kv_cache_3d,
|
||||
swa_metadata.slot_mapping,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
self._flashinfer_fp8_kv_scale,
|
||||
@@ -1497,7 +1535,14 @@ class DeepseekV4Indexer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
rotary_emb: nn.Module,
|
||||
qr_scale: torch.Tensor | None = None,
|
||||
*,
|
||||
cache_latent: torch.Tensor | None = None,
|
||||
cache_positions: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
|
||||
if cache_latent is None:
|
||||
cache_latent = latent
|
||||
if cache_positions is None:
|
||||
cache_positions = positions
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if isinstance(attn_metadata, dict):
|
||||
indexer_metadata = cast(Any, attn_metadata[self.k_cache.prefix])
|
||||
@@ -1508,7 +1553,7 @@ class DeepseekV4Indexer(nn.Module):
|
||||
# candidates num smaller than topk, every candidate is selected
|
||||
# but we still need to build k cache
|
||||
if self.owns_k:
|
||||
self._produce_k(latent, positions, rotary_emb)
|
||||
self._produce_k(cache_latent, cache_positions, rotary_emb)
|
||||
assert self.topk_indices_buffer is not None
|
||||
num_tokens = (
|
||||
indexer_metadata.num_decode_tokens
|
||||
@@ -1528,7 +1573,7 @@ class DeepseekV4Indexer(nn.Module):
|
||||
if self.owns_k:
|
||||
# K write must land before indexer_op reads the cache
|
||||
# (skip_k_cache_insert=True).
|
||||
self._produce_k(latent, positions, rotary_emb)
|
||||
self._produce_k(cache_latent, cache_positions, rotary_emb)
|
||||
|
||||
return self.forward_q(qr, qr_scale, indexer_weights, positions, rotary_emb)
|
||||
|
||||
|
||||
@@ -30,6 +30,8 @@ def fused_save_compress_norm(
|
||||
rms_norm_eps: float,
|
||||
compress_ratio: int,
|
||||
latent_out: torch.Tensor,
|
||||
prev_rows: torch.Tensor | None = None,
|
||||
prev_row_indices: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Pool each closed group into a normalized BF16 latent; save FP32 states.
|
||||
|
||||
@@ -58,6 +60,11 @@ def fused_save_compress_norm(
|
||||
rms_norm_eps: RMSNorm epsilon.
|
||||
compress_ratio: Group size, either 1 or 2.
|
||||
latent_out: BF16 [tokens, 512], written only at valid group boundaries.
|
||||
prev_rows: FP32 [rows, 1024] rows gathered from all PCP ranks (CR2). A
|
||||
chunk reads its predecessor here where ``prev_row_indices`` is set,
|
||||
and the ring is left to ``save_ring_rows``.
|
||||
prev_row_indices: [num_reqs] row in ``prev_rows`` holding the token
|
||||
before each chunk's first token, or -1 to read the ring.
|
||||
|
||||
"""
|
||||
assert compress_ratio in (1, 2)
|
||||
@@ -82,6 +89,11 @@ def fused_save_compress_norm(
|
||||
state_cache.shape[1],
|
||||
)
|
||||
num_reqs = query_start_loc.numel() - 1
|
||||
if prev_rows is not None:
|
||||
assert prev_row_indices is not None
|
||||
assert prev_row_indices.numel() >= num_reqs
|
||||
assert prev_rows.dtype == torch.float32 and prev_rows.shape[1] == 1024
|
||||
assert prev_rows.stride(1) == 1 and prev_rows.stride(0) % 16 == 0
|
||||
else:
|
||||
state_cache = query_start_loc = token_to_req_indices = None
|
||||
state_stride = state_row_stride = state_block = 1
|
||||
@@ -90,6 +102,8 @@ def fused_save_compress_norm(
|
||||
assert num_tokens <= min(kv_score.shape[0], positions.numel())
|
||||
if num_tokens == 0:
|
||||
return
|
||||
has_prev_rows = prev_rows is not None
|
||||
prev_row_stride = prev_rows.stride(0) if prev_rows is not None else 1
|
||||
grid = num_reqs + triton.cdiv(num_tokens, compress_ratio)
|
||||
_fused_save_compress_norm_kernel[(grid,)](
|
||||
kv_score,
|
||||
@@ -100,15 +114,19 @@ def fused_save_compress_norm(
|
||||
token_to_req_indices,
|
||||
rms_norm_weight,
|
||||
latent_out,
|
||||
prev_rows if has_prev_rows else kv_score,
|
||||
prev_row_indices if has_prev_rows else slot_mapping,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
RAW_STRIDE=kv_score.stride(0),
|
||||
PREV_ROW_STRIDE=prev_row_stride,
|
||||
STATE_STRIDE=state_stride,
|
||||
STATE_ROW_STRIDE=state_row_stride,
|
||||
STATE_BLOCK=state_block,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
EPS=rms_norm_eps,
|
||||
JOIN_ROW_PTRS=_JOIN_ROW_PTRS,
|
||||
HAS_PREV_ROWS=has_prev_rows,
|
||||
num_warps=4,
|
||||
**({"launch_pdl": False} if current_platform.is_cuda() else {}),
|
||||
)
|
||||
@@ -124,15 +142,19 @@ def _fused_save_compress_norm_kernel(
|
||||
req_ids,
|
||||
norm_weight,
|
||||
latent,
|
||||
prev_rows,
|
||||
prev_row_indices,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
RAW_STRIDE: tl.constexpr,
|
||||
PREV_ROW_STRIDE: tl.constexpr,
|
||||
STATE_STRIDE: tl.constexpr,
|
||||
STATE_ROW_STRIDE: tl.constexpr,
|
||||
STATE_BLOCK: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
EPS: tl.constexpr,
|
||||
JOIN_ROW_PTRS: tl.constexpr,
|
||||
HAS_PREV_ROWS: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
d = tl.arange(0, 512)
|
||||
@@ -152,6 +174,10 @@ def _fused_save_compress_norm_kernel(
|
||||
if (position + 1) % 2 == 0:
|
||||
ring = state + (state_slot // STATE_BLOCK).to(tl.int64) * STATE_STRIDE
|
||||
prev = ring + ((position - 1) % STATE_BLOCK) * STATE_ROW_STRIDE
|
||||
if HAS_PREV_ROWS:
|
||||
prev_idx = tl.load(prev_row_indices + pid)
|
||||
if prev_idx >= 0:
|
||||
prev = prev_rows + prev_idx.to(tl.int64) * PREV_ROW_STRIDE
|
||||
current = raw + start.to(tl.int64) * RAW_STRIDE
|
||||
if JOIN_ROW_PTRS:
|
||||
# Joining pointers hides row alignment from Triton; restate it.
|
||||
@@ -177,19 +203,20 @@ def _fused_save_compress_norm_kernel(
|
||||
weight_prev + weight_current
|
||||
)
|
||||
_store_latent(pooled, start, norm_weight, latent, EPS)
|
||||
num_rows = tl.minimum(end - start, STATE_BLOCK)
|
||||
for k in tl.range(0, num_rows):
|
||||
token = end - num_rows + k
|
||||
slot = tl.load(state_slots + token)
|
||||
if slot >= 0:
|
||||
row = (
|
||||
state
|
||||
+ (slot // STATE_BLOCK).to(tl.int64) * STATE_STRIDE
|
||||
+ (slot % STATE_BLOCK) * STATE_ROW_STRIDE
|
||||
)
|
||||
src = raw + token.to(tl.int64) * RAW_STRIDE
|
||||
tl.store(row + d, tl.load(src + d))
|
||||
tl.store(row + 512 + d, tl.load(src + 512 + d))
|
||||
if not HAS_PREV_ROWS:
|
||||
num_rows = tl.minimum(end - start, STATE_BLOCK)
|
||||
for k in tl.range(0, num_rows):
|
||||
token = end - num_rows + k
|
||||
slot = tl.load(state_slots + token)
|
||||
if slot >= 0:
|
||||
row = (
|
||||
state
|
||||
+ (slot // STATE_BLOCK).to(tl.int64) * STATE_STRIDE
|
||||
+ (slot % STATE_BLOCK) * STATE_ROW_STRIDE
|
||||
)
|
||||
src = raw + token.to(tl.int64) * RAW_STRIDE
|
||||
tl.store(row + d, tl.load(src + d))
|
||||
tl.store(row + 512 + d, tl.load(src + 512 + d))
|
||||
return
|
||||
|
||||
# Group program: one token (ratio 1) or the token of the pair that ends
|
||||
@@ -212,6 +239,50 @@ def _fused_save_compress_norm_kernel(
|
||||
_store_latent(pooled, t, norm_weight, latent, EPS)
|
||||
|
||||
|
||||
def save_ring_rows(
|
||||
rows: torch.Tensor, slot_mapping: torch.Tensor, state_cache: torch.Tensor
|
||||
) -> None:
|
||||
"""Store FP32 [kv, score] rows to their ring slots, skipping slot -1."""
|
||||
assert rows.dtype == torch.float32 and rows.shape[1] == 1024
|
||||
assert rows.stride(1) == 1 and state_cache.stride(2) == 1
|
||||
assert slot_mapping.numel() == rows.shape[0]
|
||||
if rows.shape[0] == 0:
|
||||
return
|
||||
_save_ring_rows_kernel[(rows.shape[0],)](
|
||||
rows,
|
||||
slot_mapping,
|
||||
state_cache,
|
||||
ROW_STRIDE=rows.stride(0),
|
||||
STATE_STRIDE=state_cache.stride(0),
|
||||
STATE_ROW_STRIDE=state_cache.stride(1),
|
||||
STATE_BLOCK=state_cache.shape[1],
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _save_ring_rows_kernel(
|
||||
rows,
|
||||
slot_mapping,
|
||||
state,
|
||||
ROW_STRIDE: tl.constexpr,
|
||||
STATE_STRIDE: tl.constexpr,
|
||||
STATE_ROW_STRIDE: tl.constexpr,
|
||||
STATE_BLOCK: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
slot = tl.load(slot_mapping + pid)
|
||||
if slot < 0:
|
||||
return
|
||||
d = tl.arange(0, 1024)
|
||||
dst = (
|
||||
state
|
||||
+ (slot // STATE_BLOCK).to(tl.int64) * STATE_STRIDE
|
||||
+ (slot % STATE_BLOCK) * STATE_ROW_STRIDE
|
||||
)
|
||||
tl.store(dst + d, tl.load(rows + pid.to(tl.int64) * ROW_STRIDE + d))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_latent(pooled, t, norm_weight, latent, EPS: tl.constexpr):
|
||||
d = tl.arange(0, 512)
|
||||
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import get_pcp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -15,6 +16,7 @@ from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
fused_save_compress_norm,
|
||||
rope_quant_insert,
|
||||
save_ring_rows,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
@@ -36,6 +38,10 @@ class CompressorBackend(AttentionBackend):
|
||||
def get_name() -> str:
|
||||
return "CompressorBackend"
|
||||
|
||||
@classmethod
|
||||
def supports_pcp(cls) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
||||
return [MultipleOf(1)]
|
||||
@@ -55,6 +61,12 @@ class CompressorMetadata:
|
||||
slot_mapping: torch.Tensor
|
||||
query_start_loc: torch.Tensor # [num_reqs + 1]
|
||||
token_to_req_indices: torch.Tensor # [num_tokens]
|
||||
# PCP only: every rank gathers the last row of each rank's local requests.
|
||||
gather_tokens: torch.Tensor | None = None # [rows] this rank's rows
|
||||
# [pcp * rows] ring slot of each request's newest row if it is open, else -1.
|
||||
ring_write_slots: torch.Tensor | None = None
|
||||
# [num_reqs] gathered row preceding each local chunk, or -1 to use the ring.
|
||||
prev_row_indices: torch.Tensor | None = None
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["block_table_stride", "num_actual_tokens", "num_tokens"])
|
||||
@@ -91,6 +103,7 @@ class CompressorMetadataBuilder(AttentionMetadataBuilder):
|
||||
super().__init__(*args, **kwargs)
|
||||
assert isinstance(self.kv_cache_spec, CircularBufferSpec)
|
||||
self.capacity = self.kv_cache_spec.block_size
|
||||
self.pcp_size = self.vllm_config.parallel_config.prefill_context_parallel_size
|
||||
max_num_batched_tokens = (
|
||||
self.vllm_config.scheduler_config.max_num_batched_tokens
|
||||
)
|
||||
@@ -100,6 +113,8 @@ class CompressorMetadataBuilder(AttentionMetadataBuilder):
|
||||
self.slot_mapping_buffer = torch.empty(
|
||||
max_num_batched_tokens, dtype=torch.int64, device=self.device
|
||||
)
|
||||
# The ring keeps only the open row under PCP, too few for spec decoding.
|
||||
assert self.pcp_size == 1 or self.vllm_config.speculative_config is None
|
||||
|
||||
def build(
|
||||
self,
|
||||
@@ -107,7 +122,7 @@ class CompressorMetadataBuilder(AttentionMetadataBuilder):
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> CompressorMetadata:
|
||||
num_tokens = common_attn_metadata.slot_mapping.numel()
|
||||
num_tokens = common_attn_metadata.slot_mapping.numel() // self.pcp_size
|
||||
positions = common_attn_metadata.positions
|
||||
assert positions is not None
|
||||
token_to_req_indices = common_attn_metadata.token_to_req_indices(
|
||||
@@ -126,11 +141,66 @@ class CompressorMetadataBuilder(AttentionMetadataBuilder):
|
||||
CAPACITY=self.capacity,
|
||||
BLOCK=256,
|
||||
)
|
||||
return CompressorMetadata(
|
||||
metadata = CompressorMetadata(
|
||||
slot_mapping=slot_mapping,
|
||||
query_start_loc=common_attn_metadata.query_start_loc,
|
||||
token_to_req_indices=token_to_req_indices,
|
||||
)
|
||||
if self.pcp_size > 1:
|
||||
self._link_pcp_chunks(metadata, common_attn_metadata.num_reqs, positions)
|
||||
return metadata
|
||||
|
||||
def _link_pcp_chunks(
|
||||
self, metadata: CompressorMetadata, num_reqs: int, positions: torch.Tensor
|
||||
) -> None:
|
||||
"""Link ratio-2 groups split across PCP chunks and pick the ring writes.
|
||||
|
||||
Every rank gathers ``(ring block, position)`` of each local request's
|
||||
last token; the ring block names a request on every rank. Sorted, they
|
||||
give each local chunk's predecessor row and each request's newest row.
|
||||
"""
|
||||
num_tokens = metadata.slot_mapping.numel()
|
||||
# Equal on every rank, so the gathers line up.
|
||||
num_rows = min(num_tokens, 2 * self.vllm_config.scheduler_config.max_num_seqs)
|
||||
num_reqs = min(num_reqs, num_rows)
|
||||
query_start_loc = metadata.query_start_loc.long()
|
||||
starts = query_start_loc[:num_reqs].clamp(max=num_tokens - 1)
|
||||
last = (query_start_loc[1 : num_reqs + 1] - 1).clamp(min=0)
|
||||
nonempty = query_start_loc[1 : num_reqs + 1] > query_start_loc[:num_reqs]
|
||||
|
||||
def key(slots: torch.Tensor, pos: torch.Tensor) -> torch.Tensor:
|
||||
return torch.where(slots >= 0, (slots // self.capacity) << 32 | pos, -1)
|
||||
|
||||
local_keys = torch.full((num_rows,), -1, dtype=torch.int64, device=self.device)
|
||||
local_keys[:num_reqs] = key(
|
||||
torch.where(nonempty, metadata.slot_mapping[last], -1), positions[last]
|
||||
)
|
||||
sorted_keys, order = get_pcp_group().all_gather(local_keys, dim=0).sort()
|
||||
|
||||
blocks, sorted_positions = sorted_keys >> 32, sorted_keys & 0xFFFFFFFF
|
||||
newest = torch.ones_like(sorted_keys, dtype=torch.bool)
|
||||
newest[:-1] = blocks[:-1] != blocks[1:]
|
||||
is_open = (sorted_keys >= 0) & newest & (sorted_positions % 2 == 0)
|
||||
ring_write_slots = torch.empty_like(sorted_keys)
|
||||
ring_write_slots[order] = torch.where(
|
||||
is_open, blocks * self.capacity + sorted_positions % self.capacity, -1
|
||||
)
|
||||
|
||||
first_positions = positions[starts]
|
||||
wanted = key(metadata.slot_mapping[starts], first_positions - 1)
|
||||
found = torch.searchsorted(sorted_keys, wanted).clamp_(max=order.numel() - 1)
|
||||
has_prev = (
|
||||
nonempty
|
||||
& (wanted >= 0)
|
||||
& (first_positions % 2 == 1)
|
||||
& (sorted_keys[found] == wanted)
|
||||
)
|
||||
|
||||
gather_tokens = torch.zeros_like(local_keys)
|
||||
gather_tokens[:num_reqs] = last
|
||||
metadata.gather_tokens = gather_tokens
|
||||
metadata.ring_write_slots = ring_write_slots
|
||||
metadata.prev_row_indices = torch.where(has_prev, order[found], -1)
|
||||
|
||||
|
||||
class CompressorStateCache(torch.nn.Module, AttentionLayerBase):
|
||||
@@ -205,6 +275,8 @@ class DeepseekCompressor(nn.Module):
|
||||
self.rotate = rotate
|
||||
self.prefix = prefix
|
||||
self.k_cache_prefix = k_cache_prefix
|
||||
self.pcp_size = vllm_config.parallel_config.prefill_context_parallel_size
|
||||
self.pcp_rank = get_pcp_group().rank_in_group if self.pcp_size > 1 else 0
|
||||
# Ratio 1 pools single tokens, so the checkpoint carries no gate.
|
||||
self.has_gate = compress_ratio > 1
|
||||
|
||||
@@ -256,15 +328,20 @@ class DeepseekCompressor(nn.Module):
|
||||
) -> torch.Tensor | None:
|
||||
"""Save states and return the BF16 latent for cache insertion and indexing.
|
||||
|
||||
Only valid group-boundary rows are written.
|
||||
Only valid group-boundary rows are written. Under PCP each rank pools
|
||||
its own rows; the last row of every rank's chunks is gathered, both to
|
||||
close groups split across chunks and to keep every rank's ring equal.
|
||||
"""
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if not isinstance(attn_metadata, dict):
|
||||
return None
|
||||
|
||||
prev_rows = prev_row_indices = ring_write_slots = None
|
||||
if self.state_cache is None:
|
||||
state_cache = query_start_loc = token_to_req_indices = None
|
||||
slot_mapping = cast(Any, attn_metadata[self.k_cache_prefix]).slot_mapping
|
||||
if self.pcp_size > 1:
|
||||
slot_mapping = slot_mapping.chunk(self.pcp_size)[self.pcp_rank]
|
||||
else:
|
||||
state_metadata = cast(
|
||||
CompressorMetadata, attn_metadata[self.state_cache.prefix]
|
||||
@@ -273,6 +350,13 @@ class DeepseekCompressor(nn.Module):
|
||||
slot_mapping = state_metadata.slot_mapping
|
||||
query_start_loc = state_metadata.query_start_loc
|
||||
token_to_req_indices = state_metadata.token_to_req_indices
|
||||
if self.pcp_size > 1:
|
||||
assert state_metadata.gather_tokens is not None
|
||||
prev_rows = get_pcp_group().all_gather(
|
||||
kv_score[state_metadata.gather_tokens], dim=0
|
||||
)
|
||||
prev_row_indices = state_metadata.prev_row_indices
|
||||
ring_write_slots = state_metadata.ring_write_slots
|
||||
|
||||
latent = torch.empty(
|
||||
kv_score.shape[0],
|
||||
@@ -291,7 +375,12 @@ class DeepseekCompressor(nn.Module):
|
||||
self.rms_norm_eps,
|
||||
self.compress_ratio,
|
||||
latent,
|
||||
prev_rows=prev_rows,
|
||||
prev_row_indices=prev_row_indices,
|
||||
)
|
||||
if prev_rows is not None:
|
||||
assert state_cache is not None and ring_write_slots is not None
|
||||
save_ring_rows(prev_rows, ring_write_slots, state_cache)
|
||||
return latent
|
||||
|
||||
def insert_cache(
|
||||
|
||||
@@ -42,6 +42,10 @@ class DeepseekSparseSWAFlashMLAMetadataBuilder(DeepseekV41SparseSWAMetadataBuild
|
||||
|
||||
|
||||
class DeepseekSparseSWAFlashMLABackend(DeepseekSparseSWABackend):
|
||||
@classmethod
|
||||
def supports_pcp(cls) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[DeepseekSparseSWAFlashMLAMetadataBuilder]:
|
||||
return DeepseekSparseSWAFlashMLAMetadataBuilder
|
||||
@@ -132,6 +136,14 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
|
||||
)
|
||||
assert swa_metadata is not None
|
||||
|
||||
if self.use_pcp:
|
||||
num_tokens = (
|
||||
swa_metadata.num_decode_tokens + swa_metadata.num_prefill_tokens
|
||||
)
|
||||
output[num_tokens:].zero_()
|
||||
if num_tokens == 0:
|
||||
return
|
||||
|
||||
swa_only = self.compress_ratio == 0
|
||||
# SWA-only layers (compress_ratio == 0) don't have their own KV cache
|
||||
# allocation; consumers read the kv source's cache, which is bound on
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_engram_dp_size,
|
||||
get_pcp_group,
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
get_tp_group,
|
||||
@@ -204,6 +205,17 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
FlashMLA path.
|
||||
"""
|
||||
backend = vllm_config.attention_config.backend
|
||||
if vllm_config.parallel_config.prefill_context_parallel_size > 1:
|
||||
if backend not in (
|
||||
None,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE_DSV4,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE_DSV41,
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"DeepSeek-V4.1 PCP requires FlashMLA sparse attention."
|
||||
)
|
||||
return DeepseekV4FlashMLAAttention
|
||||
device_capability = current_platform.get_device_capability()
|
||||
if backend in (
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE,
|
||||
@@ -684,6 +696,21 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
if self.parallel_config.prefill_context_parallel_size > 1 and (
|
||||
self.parallel_config.pipeline_parallel_size != 1
|
||||
or self.parallel_config.data_parallel_size != 1
|
||||
or self.parallel_config.decode_context_parallel_size != 1
|
||||
or self.parallel_config.use_ubatching
|
||||
or vllm_config.speculative_config is not None
|
||||
or vllm_config.kv_transfer_config is not None
|
||||
or vllm_config.cache_config.enable_prefix_caching
|
||||
or vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"DeepSeek-V4.1 PCP currently requires PP=DP=DCP=1, with no "
|
||||
"microbatching, speculative decoding, KV transfer, prefix "
|
||||
"caching, or MegaMoE."
|
||||
)
|
||||
self.use_native_mega_moe = (
|
||||
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
|
||||
)
|
||||
@@ -1019,6 +1046,13 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
hidden_states = sp_shard(hidden_states)
|
||||
input_ids = sp_shard(input_ids)
|
||||
|
||||
if (
|
||||
input_ids is not None
|
||||
and self.parallel_config.prefill_context_parallel_size > 1
|
||||
and not self.parallel_config.enable_expert_parallel
|
||||
):
|
||||
input_ids = get_pcp_group().all_gather(input_ids, dim=0)
|
||||
|
||||
residual, post_mix, res_mix = None, None, None
|
||||
pre_mix: torch.Tensor | None = None
|
||||
if not get_pp_group().is_first_rank:
|
||||
|
||||
@@ -41,7 +41,8 @@ from vllm.v1.worker.utils import AttentionGroup
|
||||
def _gather_lookback_kernel(
|
||||
lookback_ptr,
|
||||
idx_mapping_ptr,
|
||||
num_computed_tokens_ptr,
|
||||
query_start_loc_ptr,
|
||||
positions_ptr,
|
||||
all_token_ids_ptr,
|
||||
all_token_ids_stride,
|
||||
num_reqs,
|
||||
@@ -52,13 +53,16 @@ def _gather_lookback_kernel(
|
||||
batch_idx = tl.program_id(0)
|
||||
in_batch = batch_idx < num_reqs
|
||||
req_state_idx = tl.load(idx_mapping_ptr + batch_idx, mask=in_batch, other=0)
|
||||
num_computed = tl.load(num_computed_tokens_ptr + req_state_idx)
|
||||
begin = tl.load(query_start_loc_ptr + batch_idx, mask=in_batch, other=0)
|
||||
end = tl.load(query_start_loc_ptr + batch_idx + 1, mask=in_batch, other=0)
|
||||
in_batch = in_batch & (end > begin)
|
||||
num_computed = tl.load(positions_ptr + begin, mask=in_batch, other=0)
|
||||
|
||||
offs = tl.arange(0, BLOCK_DEPTH)
|
||||
pos = num_computed - 1 - offs
|
||||
valid = in_batch & (offs < DEPTH) & (pos >= 0)
|
||||
ids = tl.load(
|
||||
all_token_ids_ptr + req_state_idx * all_token_ids_stride + pos,
|
||||
all_token_ids_ptr + req_state_idx.to(tl.int64) * all_token_ids_stride + pos,
|
||||
mask=valid,
|
||||
other=-1,
|
||||
)
|
||||
@@ -200,8 +204,11 @@ class DeepseekV41ModelState(DefaultModelState):
|
||||
self.lookback_token_ids: torch.Tensor | None = None
|
||||
if depth > 0:
|
||||
# Persistent so a captured graph can read it on replay.
|
||||
max_num_reqs = self.max_num_reqs
|
||||
if vllm_config.parallel_config.prefill_context_parallel_size > 1:
|
||||
max_num_reqs *= 2
|
||||
self.lookback_token_ids = torch.full(
|
||||
(self.max_num_reqs, depth), -1, dtype=torch.int32, device=device
|
||||
(max_num_reqs, depth), -1, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
# Per request state index; batches gather from it (see prepare_attn).
|
||||
@@ -287,7 +294,8 @@ class DeepseekV41ModelState(DefaultModelState):
|
||||
_gather_lookback_kernel[(window.shape[0],)](
|
||||
window,
|
||||
input_batch.idx_mapping,
|
||||
req_states.num_computed_tokens.gpu,
|
||||
input_batch.query_start_loc,
|
||||
input_batch.positions,
|
||||
all_token_ids,
|
||||
all_token_ids.stride(0),
|
||||
input_batch.idx_mapping.shape[0],
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed import get_pcp_group
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backend import (
|
||||
@@ -24,6 +25,7 @@ from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
_LAYER_TYPE_C1A,
|
||||
_LAYER_TYPE_C2A,
|
||||
_LAYER_TYPE_SWAONLY,
|
||||
DeepseekSparseSWAMetadata,
|
||||
DeepseekSparseSWAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
@@ -60,6 +62,12 @@ class DeepseekV41SparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.pcp_size = self.vllm_config.parallel_config.prefill_context_parallel_size
|
||||
self.pcp_rank = get_pcp_group().rank_in_group if self.pcp_size > 1 else 0
|
||||
if self.pcp_size > 1:
|
||||
self.no_replay_start = self.no_replay_start.new_zeros(
|
||||
2 * self.vllm_config.scheduler_config.max_num_seqs + 1
|
||||
)
|
||||
compress_ratios = getattr(
|
||||
self.vllm_config.model_config.hf_config, "compress_ratios", None
|
||||
) or [0]
|
||||
@@ -67,6 +75,24 @@ class DeepseekV41SparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
|
||||
deepseek_v41_layer_type(int(ratio)) for ratio in compress_ratios
|
||||
}
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
replay_start: torch.Tensor | None = None,
|
||||
) -> DeepseekSparseSWAMetadata:
|
||||
cm = common_attn_metadata
|
||||
if self.pcp_size > 1:
|
||||
local_slots = cm.slot_mapping.chunk(self.pcp_size)[self.pcp_rank]
|
||||
cm = cm.replace(slot_mapping=local_slots)
|
||||
metadata = super().build(common_prefix_len, cm, fast_build, replay_start)
|
||||
if self.pcp_size > 1:
|
||||
metadata.cache_slot_mapping = common_attn_metadata.slot_mapping
|
||||
assert cm.positions is not None
|
||||
metadata.cache_positions = get_pcp_group().all_gather(cm.positions, dim=0)
|
||||
return metadata
|
||||
|
||||
|
||||
class DeepseekV4SparseMLABackend(AttentionBackend):
|
||||
"""DeepSeek-V4.1 sparse-MLA backend base.
|
||||
@@ -155,6 +181,8 @@ class DeepseekV4SparseMLAMetadataBuilder(
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
self.model_config = vllm_config.model_config
|
||||
self.pcp_size = vllm_config.parallel_config.prefill_context_parallel_size
|
||||
self.pcp_rank = get_pcp_group().rank_in_group if self.pcp_size > 1 else 0
|
||||
# Classify single-token queries (plus num_speculative_tokens via
|
||||
# supports_spec_as_decode=True) as decodes; longer queries go to prefill.
|
||||
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
|
||||
@@ -193,9 +221,12 @@ class DeepseekV4SparseMLAMetadataBuilder(
|
||||
|
||||
slot_mapping = cm.slot_mapping
|
||||
if self.compress_ratio > 1:
|
||||
local_slots = slot_mapping
|
||||
if self.pcp_size > 1:
|
||||
local_slots = slot_mapping.chunk(self.pcp_size)[self.pcp_rank]
|
||||
slot_mapping = get_compressed_slot_mapping(
|
||||
cm.num_actual_tokens,
|
||||
cm.slot_mapping,
|
||||
local_slots,
|
||||
cm.query_start_loc,
|
||||
cm.seq_lens,
|
||||
cm.block_table_tensor.clamp_(min=0),
|
||||
@@ -203,6 +234,10 @@ class DeepseekV4SparseMLAMetadataBuilder(
|
||||
self.compress_ratio,
|
||||
out=self.compressed_slot_mapping_buffer,
|
||||
)
|
||||
if self.pcp_size > 1:
|
||||
slot_mapping = get_pcp_group().all_gather(
|
||||
self.compressed_slot_mapping_buffer[: local_slots.numel()], dim=0
|
||||
)
|
||||
|
||||
return DeepseekV4FlashMLAMetadata(
|
||||
num_reqs=cm.num_reqs,
|
||||
@@ -223,6 +258,10 @@ class DeepseekV4FlashMLAMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
|
||||
|
||||
|
||||
class DeepseekV4FlashMLABackend(DeepseekV4SparseMLABackend):
|
||||
@classmethod
|
||||
def supports_pcp(cls) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASHMLA_SPARSE_DSV41"
|
||||
@@ -250,6 +289,10 @@ class FlashMLAMegaAttnBackend(DeepseekV4FlashMLABackend):
|
||||
"nvfp4_ds_mla", # V4.1 fp8 SWA cache + NVFP4 compressed cache
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def supports_pcp(cls) -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASHMLA_MEGA_ATTN_DSV41"
|
||||
|
||||
@@ -216,6 +216,9 @@ class DeepseekSparseSWAMetadata:
|
||||
# window attention may read (SWA bounded replay);
|
||||
# zeros when nothing replays.
|
||||
replay_start: torch.Tensor | None = None
|
||||
# PCP cache writes use the gathered slots; queries use rank-local slots.
|
||||
cache_slot_mapping: torch.Tensor | None = None
|
||||
cache_positions: torch.Tensor | None = None
|
||||
|
||||
# Number of decode/prefill requests/tokens (batch is reordered: decodes first)
|
||||
num_decodes: int = 0
|
||||
|
||||
Reference in New Issue
Block a user