[ROCm][Perf][DSv4.1] Enable decoder SWA bounded replay (#59039)

Signed-off-by: Frida Andersson <fanderss@amd.com>
This commit is contained in:
frida-andersson
2026-10-13 02:19:14 +00:00
committed by GitHub
parent 69d2b6f82d
commit c077dc663d
7 changed files with 705 additions and 43 deletions
@@ -290,6 +290,81 @@ def test_idle_dp_rank_dummy_trims_with_its_peers(dp_state):
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [1, 129]
def test_zero_prefix_window_warms_gather_and_skips_pad(monkeypatch):
"""ROCm leaves the prefix-cache window at 0. Replay still needs replay_start
and a warmed gather kernel. The pad kernel stays on the prefix-cache path."""
import vllm.models.deepseek_v41.nvidia.model_state as model_state
calls = {"pad_warmup": 0, "gather_warmup": 0, "pad_launch": 0}
def pad_warmup(*args, **kwargs):
calls["pad_warmup"] += 1
def gather_warmup(*args, **kwargs):
calls["gather_warmup"] += 1
class _Pad:
warmup = staticmethod(pad_warmup)
def __getitem__(self, grid):
def launch(*args, **kwargs):
calls["pad_launch"] += 1
return launch
monkeypatch.setattr(model_state, "_pad_replayed_slots_kernel", _Pad())
monkeypatch.setattr(
model_state._gather_replay_batch_kernel, "warmup", gather_warmup
)
cfg = MagicMock()
cfg.model_config.enable_prompt_embeds = False
cfg.model_config.uses_mrope = False
cfg.model_config.is_multimodal_model = False
cfg.scheduler_config.max_num_seqs = 16
cfg.scheduler_config.max_num_batched_tokens = 1024
cfg.parallel_config.data_parallel_size = 1
cfg.compilation_config.fast_moe_cold_start = False
cfg.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
layers = SimpleNamespace(window=WINDOW, replay_batch=None, metadata_prefixes=set())
model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers)
groups = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(
layer_names=["swa"],
kv_cache_spec=SimpleNamespace(
prefix_cacheable=False, prefix_replay_tokens=0
),
),
SimpleNamespace(
layer_names=["mla"],
kv_cache_spec=SimpleNamespace(
prefix_cacheable=True, prefix_replay_tokens=0
),
),
]
)
def prepare_attn(self, input_batch, cg_mode, block_tables, slot_mappings, *a, **kw):
return {"swa": kw["model_specific_attn_metadata"]}
monkeypatch.setattr(DefaultModelState, "prepare_attn", prepare_attn)
state = DeepseekV41ModelState(cfg, model, None, DEVICE)
state._replay_start_np[0] = 16
batch = _input_batch([8], [24], [True])
state.prepare_attn(
batch,
CUDAGraphMode.NONE,
BLOCK_TABLES,
_slot_mappings(batch.num_tokens_after_padding),
[],
groups,
)
assert state._replay[0] == 0
assert calls == {"pad_warmup": 0, "gather_warmup": 1, "pad_launch": 0}
assert state._replay_start[:1].tolist() == [16]
def test_dp_ranks_pad_to_one_replay_graph(dp_state):
"""Every rank pads its replay batch to the graph fitting the largest rank's."""
dp_state.replay_cudagraphs = MagicMock()