[Perf][DSv4.1] Trim the last KV-source layer in decoder replay (#59894)

Signed-off-by: Canlin <canlinguosdu@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Juntian Liu <Juntianl777@gmail.com>
This commit is contained in:
Canlin Guo
2026-10-09 06:07:49 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Juntian Liu
parent 8cbd5d0300
commit ad73a4740d
6 changed files with 219 additions and 75 deletions
@@ -67,10 +67,8 @@ def _states(num_tokens: int):
)
def test_run_gathers_states_and_realigns_shared_indexer_buffers():
topk = torch.arange(NUM_TOKENS * 4, device=DEVICE).view(NUM_TOKENS, 4).int()
candidates = torch.arange(NUM_TOKENS * 3, device=DEVICE).view(NUM_TOKENS, 3).int()
topk_before, candidates_before = topk.clone(), candidates.clone()
def test_run_gathers_states():
"""Every row's KV is written first; the layers then run on the replay rows."""
seen = {}
def run_layers(hidden_states, positions, input_ids, pre_mix, post, res, residual):
@@ -80,7 +78,8 @@ def test_run_gathers_states_and_realigns_shared_indexer_buffers():
seen["is_padding"] = replay_context.is_padding
return residual, pre_mix
layers = DecoderReplayLayers(WINDOW, run_layers, [topk, candidates], set())
write_batch_kv = MagicMock()
layers = DecoderReplayLayers(WINDOW, run_layers, write_batch_kv, set(), "swa")
states = _states(NUM_TOKENS)
hidden, residual = states[0], states[-1]
full, sub = object(), object()
@@ -90,11 +89,10 @@ def test_run_gathers_states_and_realigns_shared_indexer_buffers():
outputs = layers(*states)
assert get_forward_context() is context
write_batch_kv.assert_called_once_with(*states)
rows = torch.tensor(REPLAY_ROWS, device=DEVICE)
assert torch.equal(seen["hidden_states"], hidden[rows])
assert seen["attn_metadata"] is sub and seen["is_padding"] is None
assert torch.equal(topk[: len(REPLAY_ROWS)], topk_before[rows])
assert torch.equal(candidates[: len(REPLAY_ROWS)], candidates_before[rows])
assert len(outputs) == 2 and outputs[0].shape == residual.shape
assert torch.equal(outputs[0][rows], residual[rows])
assert outputs[0][1:173].abs().sum() == 0
@@ -107,12 +105,14 @@ def test_no_replay_batch_runs_the_whole_batch():
seen["attn_metadata"] = get_forward_context().attn_metadata
return (hidden_states,)
layers = DecoderReplayLayers(WINDOW, run_layers, [], set())
write_batch_kv = MagicMock()
layers = DecoderReplayLayers(WINDOW, run_layers, write_batch_kv, set(), "swa")
states = _states(NUM_TOKENS)
full = object()
with override_forward_context(_context(full)):
(output,) = layers(*states)
assert output is states[0] and seen["attn_metadata"] is full
write_batch_kv.assert_not_called()
def _graph_context(num_tokens):
@@ -125,6 +125,25 @@ def _graph_context(num_tokens):
)
def test_piecewise_graphs_break_out_by_size():
"""A PIECEWISE graph's KV write and break are fixed at capture, when no replay
batch is set, so both follow the graph's size rather than the replay batch."""
write_batch_kv = MagicMock()
layers = DecoderReplayLayers(
WINDOW, lambda *states: ("whole",), write_batch_kv, set(), "swa"
)
layers._run_in_graph_break = lambda *states: ("break",) # type: ignore[method-assign]
states = _states(NUM_TOKENS)
with override_forward_context(_graph_context(1024)):
assert layers(*states) == ("whole",) # no threshold: no break
layers.trim_threshold = 768
assert layers(*states) == ("break",)
_set_replay_batch(layers, REPLAY_ROWS, object())
with override_forward_context(_graph_context(576)):
assert not layers.uses_replay_batch() # a replay batch alone does not break out
write_batch_kv.assert_called_once_with(*states)
def test_replay_graph_matches_eager(monkeypatch):
"""The replay graph of the next captured size pads and runs the rows."""
monkeypatch.setattr(cudagraph_utils, "get_pp_group", MagicMock)
@@ -142,7 +161,7 @@ def test_replay_graph_matches_eager(monkeypatch):
def run_layers(hidden, positions, ids, pre, post, mix, residual):
return residual + hidden[:, None] + positions[:, None, None], pre + ids[:, None]
layers = DecoderReplayLayers(4, run_layers, [], set())
layers = DecoderReplayLayers(4, run_layers, MagicMock(), set(), "swa")
manager = DecoderReplayCudaGraphManager(cfg, DEVICE, layers)
def prepare(desc):
+37 -8
View File
@@ -11,6 +11,12 @@ import pytest
import torch
from vllm.config import CUDAGraphMode
from vllm.forward_context import (
BatchDescriptor,
ForwardContext,
override_forward_context,
)
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
@@ -59,7 +65,7 @@ def state(monkeypatch):
cfg.parallel_config.data_parallel_size = 1
cfg.compilation_config.fast_moe_cold_start = False
cfg.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
layers = SimpleNamespace(window=WINDOW, replay_batch=None, metadata_prefixes=set())
layers = DecoderReplayLayers(WINDOW, MagicMock(), MagicMock(), set(), "swa_first")
model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers)
builds: list = []
@@ -72,7 +78,7 @@ def state(monkeypatch):
attn_metadata=object(),
)
builds.append(build)
return {"swa": build.attn_metadata}
return {"swa": build.attn_metadata, "swa_first": build.attn_metadata}
monkeypatch.setattr(DefaultModelState, "prepare_attn", prepare_attn)
state = DeepseekV41ModelState(cfg, model, None, DEVICE)
@@ -120,7 +126,8 @@ def _slot_mappings(num_tokens: int) -> torch.Tensor:
def _prepare(state, batch, cg_mode):
"""Runs prepare_attn; returns the replay batch and the replay build's inputs."""
"""Runs prepare_attn; returns the replay batch and the replay build's inputs.
The first replay layer's sliding-window build follows the replay build."""
state.prepare_attn(
batch,
cg_mode,
@@ -130,7 +137,7 @@ def _prepare(state, batch, cg_mode):
GROUPS,
)
replay = state.decoder_replay_layers.replay_batch
return (None, None) if replay is None else (replay, state.builds[-1])
return (None, None) if replay is None else (replay, state.builds[-2])
def test_replay_batch_keeps_each_request_window(state):
@@ -151,8 +158,16 @@ def test_replay_batch_keeps_each_request_window(state):
assert build.replay_start.tolist() == [0, 300 - WINDOW, 50]
assert torch.equal(sub.positions, batch.positions[rows])
assert torch.equal(build.slot_mappings, _slot_mappings(401)[:, rows])
# The first replay layer keeps the encoder-side window starts: it wrote
# every row's window KV.
first = state.builds[-1]
assert first.batch is sub and first.cg_mode == CUDAGraphMode.NONE
assert first.replay_start.tolist() == [0, 0, 50]
context = replay.forward_context
assert context.attn_metadata == {"swa": build.attn_metadata}
assert context.attn_metadata == {
"swa": build.attn_metadata,
"swa_first": first.attn_metadata,
}
assert torch.equal(context.slot_mapping["mla"], _slot_mappings(401)[1, rows])
assert context.dp_metadata is None and context.is_padding is None
@@ -190,7 +205,8 @@ def test_replay_batch_keeps_adaptive_verification_query_bound(state):
def test_graph_steps_trim_only_piecewise_at_threshold(state):
"""A CUDA graph keeps the layers on its whole batch, unless it is a PIECEWISE
one at the trim threshold or above; an eager step trims a batch to its rows."""
one at the trim threshold or above, as its capture decided; an eager step
trims a batch to its rows."""
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, num_tokens_after_padding=512)
for cg_mode in (CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL):
assert _prepare(state, batch, cg_mode) == (None, None)
@@ -199,8 +215,21 @@ def test_graph_steps_trim_only_piecewise_at_threshold(state):
assert replay.rows.tolist() == REPLAY_ROWS
assert build.cg_mode == CUDAGraphMode.NONE
assert build.batch.num_tokens_after_padding == len(REPLAY_ROWS)
state._trim_threshold = 256
replay, _ = _prepare(state, batch, CUDAGraphMode.PIECEWISE)
layers = state.decoder_replay_layers
graph = ForwardContext(
no_compile_layers={},
attn_metadata={},
slot_mapping={},
cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE,
batch_descriptor=BatchDescriptor(num_tokens=512),
)
for threshold in (513, 512):
layers.trim_threshold = threshold
replay, _ = _prepare(state, batch, CUDAGraphMode.PIECEWISE)
with override_forward_context(graph): # the capture's decision
assert (
layers.uses_replay_batch() == (replay is not None) == (threshold == 512)
)
assert replay is not None and replay.rows.tolist() == REPLAY_ROWS
assert _prepare(state, batch, CUDAGraphMode.FULL) == (None, None)
+28
View File
@@ -686,6 +686,34 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
)
return self._o_proj(attn_out, positions)
def forward_kv(self, positions: torch.Tensor, hidden_states: torch.Tensor) -> None:
"""Write only the KV ``forward`` writes (SWA KV, compressed KV, index K),
skipping wq_b, the indexer's top-k and attention."""
attn_metadata = get_forward_context().attn_metadata
qr_kv, kv_score, _ = self._run_parallel_input_projections(hidden_states)
_, _, kv = self._split_qkv_and_norm(qr_kv)
if isinstance(attn_metadata, dict):
# The KV half of _fused_qnorm_rope_kv_insert, bit for bit, without
# the unused query; same as dspark's _insert_context_kv.
cache = self.swa_cache_layer.kv_cache
torch.ops._C.fused_deepseek_v4_kv_rope_insert(
kv,
cache,
cast(Any, attn_metadata[self.swa_cache_layer.prefix]).slot_mapping,
positions,
self.rotary_emb.cos_sin_cache,
self.swa_cache_layer.block_size,
self._flashinfer_fp8_kv_scale
if cache.dtype == torch.float8_e4m3fn
else None,
self.kv_mxfp8,
)
if self.compressor is not None:
latent = self.compressor(kv_score, positions)
self.compressor.insert_cache(latent, positions, self.rotary_emb)
if self.indexer is not None and self.indexer.owns_k:
self.indexer._produce_k(latent, positions, self.indexer_rotary_emb)
def bind_gemm_rs(self) -> None:
"""Fuse ``wo_b`` with the sequence-parallel TP reduce-scatter.
@@ -2,8 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Decoder-side SWA bounded replay: running the replay layers on their batch.
The layers past the last KV-source layer own nothing but sliding-window KV, so
in eager prefill steps they run on each request's last ``window`` rows only.
Once the last KV-source layer has written every row's KV, it and the layers past
it (which own only SWA KV) run on each request's last ``window`` rows only.
``DeepseekV41ModelState`` prepares those rows as a sub-batch with attention
metadata and a forward context of its own, like a microbatch;
``DecoderReplayLayers`` gathers the layer inputs by its rows, runs the layers
@@ -21,6 +21,7 @@ import torch
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.forward_context import (
ForwardContext,
get_forward_context,
in_piecewise_cudagraph,
override_forward_context,
)
@@ -39,35 +40,57 @@ class ReplayBatch:
class DecoderReplayLayers:
"""Runs the replay layers on the step's replay batch.
``run_layers`` takes a batch's layer inputs and returns its per-row
outputs. ``row_buffers`` hold per-row results the source layer's indexer
publishes for the layers after it; they are compacted to the replay rows
in place. ``metadata_prefixes`` are the attention metadata keys the layers
read, so the replay batch builds only those.
``run_layers`` takes a batch's layer inputs and returns its per-row outputs;
``write_batch_kv`` takes them too and writes the first replay layer's KV for
every row. ``metadata_prefixes`` are the attention metadata keys the layers
read, so the replay batch builds only those. ``first_swa_prefix`` keys the
first replay layer's SWA metadata, built with the batch's window starts.
"""
def __init__(
self,
window: int,
run_layers: Callable[..., tuple[torch.Tensor, ...]],
row_buffers: list[torch.Tensor],
write_batch_kv: Callable[..., None],
metadata_prefixes: set[str],
first_swa_prefix: str,
) -> None:
self.window = window
self.run_layers = run_layers
self.row_buffers = row_buffers
self.write_batch_kv = write_batch_kv
self.metadata_prefixes = metadata_prefixes
self.first_swa_prefix = first_swa_prefix
# Set by the model state every step; None runs the layers on the batch.
self.replay_batch: ReplayBatch | None = None
# Set by the model state with replay graphs: PIECEWISE model graphs of
# this many tokens or more break out to the replay batch.
self.trim_threshold: int | None = None
self._break_outputs: list[torch.Tensor] | None = None
def __call__(
self, hidden_states: torch.Tensor | MoEOutput, *states: torch.Tensor | None
) -> tuple[torch.Tensor, ...]:
if self.replay_batch is not None and in_piecewise_cudagraph():
return self._run_in_graph_break(hidden_states, *states)
if self.uses_replay_batch():
# Outside the break, so a PIECEWISE graph captures it.
self.write_batch_kv(hidden_states, *states)
if in_piecewise_cudagraph():
return self._run_in_graph_break(hidden_states, *states)
return self._run(hidden_states, *states)
def uses_replay_batch(self) -> bool:
"""Whether this forward uses the replay batch: in a PIECEWISE graph by its
size, fixed at capture (no replay batch is set then); else if one is set."""
if in_piecewise_cudagraph():
batch_descriptor = get_forward_context().batch_descriptor
assert batch_descriptor is not None
return self.graph_uses_replay_batch(batch_descriptor.num_tokens)
return self.replay_batch is not None
def graph_uses_replay_batch(self, num_tokens: int) -> bool:
"""Whether a PIECEWISE graph of ``num_tokens`` breaks out to the replay
batch; capture and the model state's runtime replay batch share it."""
return self.trim_threshold is not None and num_tokens >= self.trim_threshold
@eager_break_during_capture
def _run_in_graph_break(
self, hidden_states: torch.Tensor | MoEOutput, *states: torch.Tensor | None
@@ -91,9 +114,6 @@ class DecoderReplayLayers:
# Replay batch steps hold more than `window` tokens: no deferred MoE finalize.
assert isinstance(hidden_states, torch.Tensor)
rows = replay_batch.rows
num_rows = rows.shape[0]
for buf in self.row_buffers:
buf[:num_rows].copy_(buf.index_select(0, rows))
inputs = (hidden_states, *states)
with override_forward_context(replay_batch.forward_context):
if replay_batch.run_graph is not None:
+48 -14
View File
@@ -623,6 +623,40 @@ class DeepseekV4DecoderLayer(nn.Module):
torch.cuda.current_stream().wait_stream(mhc_stream)
return x, residual, post_mix, res_mix, ffn_pre, previous_aux
def write_kv(
self,
x: torch.Tensor | MoEOutput,
positions: torch.Tensor,
input_ids: torch.Tensor | None,
pre_mix: torch.Tensor,
post_mix: torch.Tensor,
res_mix: torch.Tensor,
residual: torch.Tensor,
) -> None:
"""Write the KV ``forward`` would for the replay layers' inputs, skipping
the rest of the layer; the replay batch then reruns its rows."""
# The replay batch rules out Engram and sequence parallel here.
assert self.engram is None and not self.use_sequence_parallel
*_, x, _, _ = mhc_shifted_post_pre(
x,
residual,
post_mix,
res_mix,
self.hc_attn_fn,
self.hc_attn_scale,
self.hc_attn_base,
self.rms_norm_eps,
self.hc_eps,
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
pre_mix=pre_mix,
norm_weight=self.attn_norm.weight,
norm_eps=self.attn_norm.variance_epsilon,
reduce_results=self.fuse_mhc_all_reduce,
)
self.attn.forward_kv(positions, x)
class DeepseekV4Model(nn.Module, EagleModelMixin):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
@@ -765,10 +799,12 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
and self._decoder_replay_supported(vllm_config, cut)
and self.layers[cut].attn.swa_cache_layer.bounded_replay
):
self.decoder_replay_start = cut + 1
assert self.layers[cut].attn.compress_ratio <= 1
# The last KV source writes every row's KV, then replays.
self.decoder_replay_start = cut
# The attention metadata keys the replay layers read.
metadata_prefixes = set()
for layer in islice(self.layers, cut + 1, self.end_layer):
for layer in islice(self.layers, cut, self.end_layer):
attn = typing.cast(DeepseekV4DecoderLayer, layer).attn
metadata_prefixes.add(attn.swa_cache_layer.prefix)
if attn.compressed_cache_prefix is not None:
@@ -778,16 +814,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
self.decoder_replay_layers = DecoderReplayLayers(
config.sliding_window,
self._run_replay_layers,
[
buf
for buf in (self.topk_indices_buffer, self.candidate_block_buffer)
if buf is not None
],
self.layers[cut].write_kv,
metadata_prefixes,
self.layers[cut].attn.swa_cache_layer.prefix,
)
logger.info_once(
"Decoder SWA bounded replay: in eager prefill steps, layers "
"Decoder SWA bounded replay: in eager prefill steps, layer %d "
"writes its KV for every token, and the rest of it and layers "
"%d-%d run on each request's last %d tokens only.",
cut,
cut + 1,
self.end_layer - 1,
config.sliding_window,
@@ -1120,9 +1155,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
res_mix: torch.Tensor,
residual: torch.Tensor,
) -> tuple[torch.Tensor, ...]:
"""The layers past the last KV source, on whatever rows they are given;
returns their output, the last FFN's pre-mix and the aux hidden states
they capture."""
"""Layers ``decoder_replay_start``.. on the given rows; returns their output,
the last FFN's pre-mix and the aux hidden states they capture."""
aux_hidden_by_layer: dict[int, torch.Tensor] = {}
hidden_states, residual, post_mix, res_mix, pre_mix = self._run_layers(
range(self.decoder_replay_start, self.end_layer),
@@ -1154,7 +1188,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
)
def _decoder_replay_supported(self, vllm_config: VllmConfig, cut: int) -> bool:
"""Whether this rank may trim the layers after ``cut``; warns when not."""
"""Whether this rank may replay from layer ``cut`` on; warns when not."""
parallel_config = vllm_config.parallel_config
spec_config = vllm_config.speculative_config
draft_config = spec_config.draft_model_config if spec_config else None
@@ -1176,8 +1210,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
"the replay-layer batch shrinks per rank, which sequence and "
"prefill-context parallelism and microbatching cannot follow"
)
elif any(i > cut for i in getattr(self.config, "engram_layer_ids", ())):
reason = "an Engram layer sits after the last KV source layer"
elif any(i >= cut for i in getattr(self.config, "engram_layer_ids", ())):
reason = "an Engram layer sits at or after the last KV source layer"
elif draft_config is not None and (
draft_window is None
or draft_window > window
+44 -30
View File
@@ -238,12 +238,13 @@ class DeepseekV41ModelState(DefaultModelState):
self._kept_kv_start = torch.zeros(
self.max_num_reqs, dtype=torch.int32, device=device
)
self._replay_attn_groups: list[list[AttentionGroup]] | None = None
# The replay layers' and the first one's SWA attention groups.
self._replay_attn_groups: (
tuple[list[list[AttentionGroup]], list[list[AttentionGroup]]] | None
) = None
layers = self.decoder_replay_layers
compilation_config = vllm_config.compilation_config
self.replay_cudagraphs: DecoderReplayCudaGraphManager | None = None
# PIECEWISE graphs of this many tokens or more break out to the replay batch.
self._trim_threshold: int | None = None
if (
layers is not None
and compilation_config.cudagraph_decoder_replay
@@ -263,7 +264,7 @@ class DeepseekV41ModelState(DefaultModelState):
self.replay_cudagraphs = DecoderReplayCudaGraphManager(
vllm_config, device, layers
)
self._trim_threshold = threshold
layers.trim_threshold = threshold
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
super().add_request(req_index, new_req_data)
@@ -454,8 +455,7 @@ class DeepseekV41ModelState(DefaultModelState):
layers.replay_batch = None
force_replay_batch = capture_desc is not None or (
cudagraph_mode == CUDAGraphMode.PIECEWISE
and self._trim_threshold is not None
and input_batch.num_tokens_after_padding >= self._trim_threshold
and layers.graph_uses_replay_batch(input_batch.num_tokens_after_padding)
)
if cudagraph_mode != CUDAGraphMode.NONE and not force_replay_batch:
return
@@ -509,17 +509,35 @@ class DeepseekV41ModelState(DefaultModelState):
kept_batch, kept_slot_mappings = self._kept_input_batch(
input_batch, slot_mappings, replay_start, kept_lens, num_padded
)
if self._replay_attn_groups is None:
self._replay_attn_groups = (
self._copy_attn_groups(attn_groups, layers.metadata_prefixes),
self._copy_attn_groups(attn_groups, {layers.first_swa_prefix}),
)
replay_groups, first_swa_groups = self._replay_attn_groups
attn_metadata = super().prepare_attn(
kept_batch,
CUDAGraphMode.NONE,
block_tables,
kept_slot_mappings,
self._replay_groups(attn_groups),
replay_groups,
kv_cache_config,
model_specific_attn_metadata=ReplayAttnMetadata(
self._kept_kv_start[:num_reqs]
),
)
# The first replay layer wrote every row's window KV: its window starts
# are the batch's.
first_swa = layers.first_swa_prefix
attn_metadata[first_swa] = super().prepare_attn(
kept_batch,
CUDAGraphMode.NONE,
block_tables,
kept_slot_mappings,
first_swa_groups,
kv_cache_config,
model_specific_attn_metadata=ReplayAttnMetadata(replay_start),
)[first_swa]
layers.replay_batch = ReplayBatch(
self._kept_rows[:num_tokens],
create_forward_context(
@@ -636,27 +654,23 @@ class DeepseekV41ModelState(DefaultModelState):
return False, None
return True, agreed[1]
def _replay_groups(
self, attn_groups: list[list[AttentionGroup]]
def _copy_attn_groups(
self, attn_groups: list[list[AttentionGroup]], prefixes: set[str]
) -> list[list[AttentionGroup]]:
"""The attention groups with metadata builders of their own, like each
microbatch's: a builder keeps the metadata it built, and the runner's
hold the batch's. Only the groups the replay layers read are built."""
if self._replay_attn_groups is None:
assert self.decoder_replay_layers is not None
prefixes = self.decoder_replay_layers.metadata_prefixes
self._replay_attn_groups = []
for groups in attn_groups:
replay_groups = []
for group in groups:
if prefixes.isdisjoint(group.layer_names):
continue
replay_group = replace(group, metadata_builders=[])
replay_group.create_metadata_builders(
self.vllm_config,
self.device,
group.metadata_builders[0].kernel_block_size,
)
replay_groups.append(replay_group)
self._replay_attn_groups.append(replay_groups)
return self._replay_attn_groups
"""Copies of the groups holding any of ``prefixes``, with builders of their
own like a microbatch's: a builder keeps the metadata it built."""
own_groups = []
for groups in attn_groups:
copies = []
for group in groups:
if prefixes.isdisjoint(group.layer_names):
continue
copy = replace(group, metadata_builders=[])
copy.create_metadata_builders(
self.vllm_config,
self.device,
group.metadata_builders[0].kernel_block_size,
)
copies.append(copy)
own_groups.append(copies)
return own_groups