[Model] Decoder-side SWA bounded replay for DeepSeek-V4.1 (#58132)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
9af952c555
commit
4b2e1cfa7b
@@ -507,20 +507,22 @@ def _check_dummy_hash_model_forward(
|
||||
assert torch.all(hashes == engram_ops.DEAD_ID)
|
||||
output = engram.embed(hashes[:, 0])
|
||||
# Trailing None is previous_aux; this stub captures no aux states.
|
||||
return output, None, None, None, None, None
|
||||
return output, hidden, hidden, hidden, hidden, None
|
||||
|
||||
model = SimpleNamespace(
|
||||
use_mega_moe=False,
|
||||
use_sequence_parallel=False,
|
||||
fuse_mhc_all_reduce=False,
|
||||
engram_hash=state,
|
||||
engram_swa_prefix="swa",
|
||||
engram_dp_shared_memory=dp_shared_memory,
|
||||
layers=[Decoder(engram=engram)],
|
||||
start_layer=0,
|
||||
end_layer=1,
|
||||
aux_hidden_state_layers=(),
|
||||
)
|
||||
model = model_ops.DeepseekV4Model.__new__(model_ops.DeepseekV4Model)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.use_mega_moe = False
|
||||
model.use_sequence_parallel = False
|
||||
model.fuse_mhc_all_reduce = False
|
||||
model.engram_hash = state
|
||||
model.engram_swa_prefix = "swa"
|
||||
model.engram_dp_shared_memory = dp_shared_memory
|
||||
model.layers = [Decoder(engram=engram)]
|
||||
model.start_layer = 0
|
||||
model.end_layer = 1
|
||||
model.decoder_replay_start = model.end_layer
|
||||
model.decoder_replay_layers = None
|
||||
model.aux_hidden_state_layers = ()
|
||||
metadata = (
|
||||
None
|
||||
if dp_rank == 1
|
||||
@@ -550,8 +552,7 @@ def _check_dummy_hash_model_forward(
|
||||
dtype=torch.int32,
|
||||
),
|
||||
):
|
||||
output = model_ops.DeepseekV4Model.forward(
|
||||
model,
|
||||
output = model(
|
||||
torch.arange(tokens, device="cuda"),
|
||||
torch.arange(tokens, device="cuda"),
|
||||
intermediate_tensors=None,
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DecoderReplayLayers runs the replay layers on the forward's replay batch."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.forward_context import (
|
||||
ForwardContext,
|
||||
get_forward_context,
|
||||
override_forward_context,
|
||||
)
|
||||
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
|
||||
|
||||
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
|
||||
|
||||
DEVICE = torch.device("cuda")
|
||||
NUM_TOKENS = 401
|
||||
HC, HIDDEN = 2, 3
|
||||
# decode (1 token), trimmed prefill (300 of 300), untrimmed prefill (100 of 300)
|
||||
WINDOW = 128
|
||||
REPLAY_ROWS = [0, *range(301 - WINDOW, 301), *range(301, 401)]
|
||||
|
||||
|
||||
def _context(metadata):
|
||||
return ForwardContext(
|
||||
no_compile_layers={},
|
||||
attn_metadata=metadata,
|
||||
slot_mapping={},
|
||||
is_padding=torch.zeros(NUM_TOKENS, dtype=torch.bool, device=DEVICE),
|
||||
)
|
||||
|
||||
|
||||
def _set_replay_batch(layers, rows, metadata):
|
||||
layers.rows = torch.tensor(rows, device=DEVICE)
|
||||
layers.forward_context = ForwardContext(
|
||||
no_compile_layers={}, attn_metadata=metadata, slot_mapping={}
|
||||
)
|
||||
|
||||
|
||||
def _states(num_tokens: int):
|
||||
"""(hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual),
|
||||
shaped as the model's: hidden states [T, H], mixes [T, HC], residual [T, HC, H]."""
|
||||
hidden = torch.arange(num_tokens, device=DEVICE, dtype=torch.float32)
|
||||
hidden = hidden[:, None].expand(num_tokens, HIDDEN).contiguous()
|
||||
mix = hidden[:, :HC].contiguous()
|
||||
residual = hidden[:, None, :].expand(num_tokens, HC, HIDDEN).contiguous()
|
||||
return (
|
||||
hidden,
|
||||
torch.arange(num_tokens, device=DEVICE),
|
||||
None,
|
||||
mix + 1,
|
||||
mix + 2,
|
||||
mix + 3,
|
||||
residual,
|
||||
)
|
||||
|
||||
|
||||
def test_run_gathers_states_and_realigns_shared_indexer_buffers():
|
||||
topk = torch.arange(NUM_TOKENS * 4, device=DEVICE).view(NUM_TOKENS, 4).int()
|
||||
candidates = torch.arange(NUM_TOKENS * 3, device=DEVICE).view(NUM_TOKENS, 3).int()
|
||||
topk_before, candidates_before = topk.clone(), candidates.clone()
|
||||
seen = {}
|
||||
|
||||
def run_layers(hidden_states, positions, input_ids, pre_mix, post, res, residual):
|
||||
seen["hidden_states"] = hidden_states.clone()
|
||||
replay_context = get_forward_context()
|
||||
seen["attn_metadata"] = replay_context.attn_metadata
|
||||
seen["is_padding"] = replay_context.is_padding
|
||||
return residual, pre_mix
|
||||
|
||||
layers = DecoderReplayLayers(WINDOW, run_layers, [topk, candidates])
|
||||
states = _states(NUM_TOKENS)
|
||||
hidden, residual = states[0], states[-1]
|
||||
full, sub = object(), object()
|
||||
_set_replay_batch(layers, REPLAY_ROWS, sub)
|
||||
context = _context(full)
|
||||
with override_forward_context(context):
|
||||
outputs = layers(*states)
|
||||
assert get_forward_context() is context
|
||||
|
||||
rows = torch.tensor(REPLAY_ROWS, device=DEVICE)
|
||||
assert torch.equal(seen["hidden_states"], hidden[rows])
|
||||
assert seen["attn_metadata"] is sub and seen["is_padding"] is None
|
||||
assert torch.equal(topk[: len(REPLAY_ROWS)], topk_before[rows])
|
||||
assert torch.equal(candidates[: len(REPLAY_ROWS)], candidates_before[rows])
|
||||
assert len(outputs) == 2 and outputs[0].shape == residual.shape
|
||||
assert torch.equal(outputs[0][rows], residual[rows])
|
||||
assert outputs[0][1:173].abs().sum() == 0
|
||||
|
||||
|
||||
def test_no_replay_batch_runs_the_whole_batch():
|
||||
seen = {}
|
||||
|
||||
def run_layers(hidden_states, *rest):
|
||||
seen["attn_metadata"] = get_forward_context().attn_metadata
|
||||
return (hidden_states,)
|
||||
|
||||
layers = DecoderReplayLayers(WINDOW, run_layers, [])
|
||||
states = _states(NUM_TOKENS)
|
||||
full = object()
|
||||
with override_forward_context(_context(full)):
|
||||
(output,) = layers(*states)
|
||||
assert output is states[0] and seen["attn_metadata"] is full
|
||||
@@ -0,0 +1,259 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The decoder-side SWA bounded replay batch DeepseekV41ModelState prepares."""
|
||||
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="the replay buffers live on the GPU"
|
||||
)
|
||||
|
||||
WINDOW = 128
|
||||
DEVICE = torch.device("cuda")
|
||||
GROUPS = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(
|
||||
layer_names=["swa"],
|
||||
kv_cache_spec=SimpleNamespace(
|
||||
prefix_cacheable=False, prefix_replay_tokens=WINDOW
|
||||
),
|
||||
),
|
||||
SimpleNamespace(
|
||||
layer_names=["mla"],
|
||||
kv_cache_spec=SimpleNamespace(
|
||||
prefix_cacheable=True, prefix_replay_tokens=0
|
||||
),
|
||||
),
|
||||
]
|
||||
)
|
||||
BLOCK_TABLES = (torch.zeros(4, 4, device=DEVICE),) * 2
|
||||
|
||||
# decode (1 token), trimmed prefill (300 of 300), untrimmed prefill (100 of 300)
|
||||
QUERY_LENS = [1, 300, 100]
|
||||
SEQ_LENS = [500, 300, 300]
|
||||
PREFILLING = [False, True, True]
|
||||
REPLAY_ROWS = [0, *range(301 - WINDOW, 301), *range(301, 401)]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def state(monkeypatch):
|
||||
"""A DeepseekV41ModelState whose attention builds are recorded, not run."""
|
||||
cfg = MagicMock()
|
||||
cfg.model_config.enable_prompt_embeds = False
|
||||
cfg.model_config.uses_mrope = False
|
||||
cfg.model_config.is_multimodal_model = False
|
||||
cfg.scheduler_config.max_num_seqs = 16
|
||||
cfg.scheduler_config.max_num_batched_tokens = 1024
|
||||
cfg.parallel_config.data_parallel_size = 1
|
||||
cfg.compilation_config.fast_moe_cold_start = False
|
||||
layers = SimpleNamespace(window=WINDOW, rows=None, forward_context=None)
|
||||
model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers)
|
||||
builds: list = []
|
||||
|
||||
def prepare_attn(self, input_batch, cg_mode, block_tables, slot_mappings, *a, **kw):
|
||||
build = SimpleNamespace(
|
||||
batch=input_batch,
|
||||
cg_mode=cg_mode,
|
||||
slot_mappings=slot_mappings.clone(),
|
||||
replay_start=kw["model_specific_attn_metadata"].replay_start,
|
||||
attn_metadata=object(),
|
||||
)
|
||||
builds.append(build)
|
||||
return {"swa": build.attn_metadata}
|
||||
|
||||
monkeypatch.setattr(DefaultModelState, "prepare_attn", prepare_attn)
|
||||
state = DeepseekV41ModelState(cfg, model, None, DEVICE)
|
||||
state.builds = builds # type: ignore[attr-defined]
|
||||
return state
|
||||
|
||||
|
||||
def _input_batch(
|
||||
query_lens: list[int],
|
||||
seq_lens: list[int],
|
||||
is_prefilling: list[bool],
|
||||
device_query_lens: list[int] | None = None,
|
||||
num_tokens_after_padding: int | None = None,
|
||||
max_query_len: int | None = None,
|
||||
) -> InputBatch:
|
||||
num_reqs, num_tokens = len(query_lens), sum(query_lens)
|
||||
num_padded = num_tokens_after_padding or num_tokens
|
||||
batch = InputBatch.make_dummy(num_reqs, num_tokens, InputBuffers(16, 1024, DEVICE))
|
||||
return replace(
|
||||
batch,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_after_padding=num_padded,
|
||||
query_start_loc=torch.tensor(
|
||||
[0, *np.cumsum(device_query_lens or query_lens)],
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
),
|
||||
query_start_loc_np=np.array([0, *np.cumsum(query_lens)], dtype=np.int32),
|
||||
seq_lens=torch.tensor(seq_lens, dtype=torch.int32, device=DEVICE),
|
||||
seq_lens_cpu_upper_bound=torch.tensor(seq_lens, dtype=torch.int32),
|
||||
is_prefilling_np=np.array(is_prefilling, dtype=np.bool_),
|
||||
positions=torch.cat(
|
||||
[torch.arange(s - q, s) for q, s in zip(query_lens, seq_lens)]
|
||||
+ [torch.zeros(num_padded - num_tokens, dtype=torch.int64)]
|
||||
).to(DEVICE),
|
||||
is_padding=torch.arange(num_padded, device=DEVICE) >= num_tokens,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
|
||||
|
||||
def _slot_mappings(num_tokens: int) -> torch.Tensor:
|
||||
return torch.stack([torch.arange(num_tokens), torch.arange(num_tokens) * 10]).to(
|
||||
DEVICE
|
||||
)
|
||||
|
||||
|
||||
def _prepare(state, batch, cg_mode):
|
||||
"""Runs prepare_attn; returns the replay batch and the replay build's inputs."""
|
||||
state.prepare_attn(
|
||||
batch,
|
||||
cg_mode,
|
||||
BLOCK_TABLES,
|
||||
_slot_mappings(batch.num_tokens_after_padding),
|
||||
[],
|
||||
GROUPS,
|
||||
)
|
||||
layers = state.decoder_replay_layers
|
||||
if layers.rows is None:
|
||||
assert layers.forward_context is None
|
||||
return None, None
|
||||
return SimpleNamespace(
|
||||
rows=layers.rows, forward_context=layers.forward_context
|
||||
), state.builds[-1]
|
||||
|
||||
|
||||
def test_replay_batch_keeps_each_request_window(state):
|
||||
state._replay_start_np[2] = 50 # the encoder-side replay start of request 2
|
||||
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
|
||||
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
|
||||
assert replay is not None
|
||||
rows = torch.tensor(REPLAY_ROWS, device=DEVICE)
|
||||
assert torch.equal(replay.rows, rows)
|
||||
sub = build.batch
|
||||
assert build.cg_mode == CUDAGraphMode.NONE
|
||||
assert sub.query_start_loc.tolist() == [0, 1, 129, 229]
|
||||
assert sub.query_start_loc_np.tolist() == [0, 1, 129, 229]
|
||||
assert sub.num_tokens == sub.num_tokens_after_padding == 229
|
||||
assert sub.max_query_len == WINDOW
|
||||
# The trimmed request's window starts at its replay window; a higher
|
||||
# encoder-side replay start stands.
|
||||
assert build.replay_start.tolist() == [0, 300 - WINDOW, 50]
|
||||
assert torch.equal(sub.positions, batch.positions[rows])
|
||||
assert torch.equal(build.slot_mappings, _slot_mappings(401)[:, rows])
|
||||
context = replay.forward_context
|
||||
assert context.attn_metadata == {"swa": build.attn_metadata}
|
||||
assert torch.equal(context.slot_mapping["mla"], _slot_mappings(401)[1, rows])
|
||||
assert context.dp_metadata is None and context.is_padding is None
|
||||
|
||||
|
||||
def test_replay_batch_keeps_device_decode_boundaries(state):
|
||||
"""Adaptive verification resizes decodes on the GPU (CPU [2, 2], device
|
||||
[1, 3]): their rows stay and the boundaries shift only by trimmed rows."""
|
||||
batch = _input_batch(
|
||||
[2, 2, 300], [10, 10, 300], PREFILLING, device_query_lens=[1, 3, 300]
|
||||
)
|
||||
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
|
||||
assert replay is not None
|
||||
assert replay.rows.tolist() == [0, 1, 2, 3, *range(304 - WINDOW, 304)]
|
||||
assert build.batch.query_start_loc.tolist() == [0, 1, 4, 4 + WINDOW]
|
||||
|
||||
|
||||
def test_prompt_logprobs_requests_keep_their_rows(state):
|
||||
"""Every prompt row of a prompt-logprobs request is read, so it never
|
||||
trims; the other prefills still do."""
|
||||
state._keeps_rows_np[1] = True
|
||||
batch = _input_batch([1, 300, 300], [500, 300, 300], PREFILLING)
|
||||
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
|
||||
assert replay is not None
|
||||
assert replay.rows.tolist() == [0, *range(1, 301), *range(601 - WINDOW, 601)]
|
||||
assert build.batch.max_query_len == 300
|
||||
|
||||
|
||||
def test_replay_batch_keeps_adaptive_verification_query_bound(state):
|
||||
"""Adaptive verification bounds the decodes' device-side query lengths
|
||||
above their CPU lengths; the replay batch keeps that bound."""
|
||||
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, max_query_len=200)
|
||||
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
|
||||
assert replay is not None and build.batch.max_query_len == 200
|
||||
|
||||
|
||||
def test_only_eager_steps_trim(state):
|
||||
"""A CUDA graph keeps the layers on its whole batch; an eager step trims a
|
||||
padded batch to its real rows."""
|
||||
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, num_tokens_after_padding=512)
|
||||
for cg_mode in (CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL):
|
||||
assert _prepare(state, batch, cg_mode) == (None, None)
|
||||
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
|
||||
assert replay is not None
|
||||
assert replay.rows.tolist() == REPLAY_ROWS
|
||||
assert build.cg_mode == CUDAGraphMode.NONE
|
||||
assert build.batch.num_tokens_after_padding == len(REPLAY_ROWS)
|
||||
|
||||
|
||||
def test_whole_batch_forwards_get_no_replay_batch(state):
|
||||
short = _input_batch([100, 100], [100, 100], [True, True])
|
||||
for cg_mode in (CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL):
|
||||
assert _prepare(state, short, cg_mode) == (None, None)
|
||||
assert len(state.builds) == 3 # the batch's own metadata only
|
||||
# Dummy batches, captures included (prepared as eager), are not prefills and
|
||||
# keep their rows.
|
||||
dummy = _input_batch(
|
||||
QUERY_LENS, SEQ_LENS, [False] * 3, num_tokens_after_padding=512
|
||||
)
|
||||
assert _prepare(state, dummy, CUDAGraphMode.NONE) == (None, None)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dp_state(state, monkeypatch):
|
||||
"""`state` on DP rank 0 of 2; `state.other` sets what rank 1 reports
|
||||
(trims, replay tokens)."""
|
||||
import vllm.models.deepseek_v41.nvidia.model_state as module
|
||||
|
||||
state.vllm_config.parallel_config.data_parallel_size = 2
|
||||
state.vllm_config.parallel_config.data_parallel_rank = 0
|
||||
state.vllm_config.parallel_config.is_moe_model = True
|
||||
state.other = (False, 0) # type: ignore[attr-defined]
|
||||
|
||||
def all_reduce(tensor, group):
|
||||
tensor[:, 1] = torch.tensor(state.other, dtype=torch.int32)
|
||||
|
||||
monkeypatch.setattr(module.dist, "all_reduce", all_reduce)
|
||||
monkeypatch.setattr(module, "get_dp_group", lambda: SimpleNamespace(cpu_group=None))
|
||||
return state
|
||||
|
||||
|
||||
def test_dp_ranks_replay_together(dp_state):
|
||||
"""A trimming rank makes the others replay too, on every rank's count."""
|
||||
short = _input_batch([100, 100], [100, 100], [True, True])
|
||||
dp_state.other = (False, 200)
|
||||
assert _prepare(dp_state, short, CUDAGraphMode.NONE) == (None, None)
|
||||
|
||||
dp_state.other = (True, 300) # rank 1 trims to 300 rows
|
||||
replay, build = _prepare(dp_state, short, CUDAGraphMode.NONE)
|
||||
assert replay is not None
|
||||
assert replay.rows.tolist() == list(range(200)) # this rank keeps every row
|
||||
dp_metadata = replay.forward_context.dp_metadata
|
||||
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [200, 300]
|
||||
|
||||
|
||||
def test_idle_dp_rank_dummy_trims_with_its_peers(dp_state):
|
||||
dummy = _input_batch([512], [512], [False])
|
||||
dp_state.other = (True, 129)
|
||||
replay, _ = _prepare(dp_state, dummy, CUDAGraphMode.NONE)
|
||||
assert replay is not None and replay.rows.shape[0] == WINDOW
|
||||
dp_metadata = replay.forward_context.dp_metadata
|
||||
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [WINDOW, 129]
|
||||
Reference in New Issue
Block a user