[Perf] Avoid more GPU<->CPU syncs on the model execution path (#51738)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-11 19:30:34 -07:00
committed by GitHub
parent 23f25aac14
commit f97e502969
15 changed files with 116 additions and 56 deletions
+2 -2
View File
@@ -248,13 +248,13 @@ def _audioflamingo3_field_config(hf_inputs: Mapping[str, torch.Tensor]):
feature_attention_mask=MultiModalFieldConfig.flat_from_sizes( feature_attention_mask=MultiModalFieldConfig.flat_from_sizes(
"audio", chunk_counts, dim=0 "audio", chunk_counts, dim=0
), ),
chunk_counts=MultiModalFieldConfig.batched("audio"), chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
) )
return dict( return dict(
audio_embeds=MultiModalFieldConfig.batched("audio"), audio_embeds=MultiModalFieldConfig.batched("audio"),
input_features=MultiModalFieldConfig.batched("audio"), input_features=MultiModalFieldConfig.batched("audio"),
feature_attention_mask=MultiModalFieldConfig.batched("audio"), feature_attention_mask=MultiModalFieldConfig.batched("audio"),
chunk_counts=MultiModalFieldConfig.batched("audio"), chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
) )
+20 -12
View File
@@ -48,6 +48,7 @@ from vllm.model_executor.models.transformers.utils import recursive_replace_line
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.outputs import LogprobsTensors from vllm.v1.outputs import LogprobsTensors
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
@@ -728,10 +729,13 @@ class DiffusionGemmaRequestStates:
max_num_reqs, canvas_length, hidden_size, dtype=torch.float32, device=device max_num_reqs, canvas_length, hidden_size, dtype=torch.float32, device=device
) )
def init_canvas(self, slot_indices_np: np.ndarray) -> None: def init_canvas(self, slot_indices: torch.Tensor) -> None:
"""Initialize canvas with random tokens for the given slots.""" """Initialize canvas with random tokens for the given slots.
n = slot_indices_np.shape[0]
self.canvas[slot_indices_np] = torch.randint( `slot_indices` must already be on device to avoid a cpu->gpu sync.
"""
n = slot_indices.shape[0]
self.canvas[slot_indices] = torch.randint(
0, 0,
self.vocab_size, self.vocab_size,
(n, self.canvas_length), (n, self.canvas_length),
@@ -740,15 +744,15 @@ class DiffusionGemmaRequestStates:
) )
def add_request(self, slot_idx: int) -> None: def add_request(self, slot_idx: int) -> None:
self.is_encoder_phase[slot_idx] = True self.is_encoder_phase[slot_idx].fill_(True)
self.init_canvas(torch.tensor([slot_idx], device=self.device)) self.init_canvas(async_tensor_h2d([slot_idx], device=self.device))
self.step[slot_idx] = 0 self.step[slot_idx].fill_(0)
self.accepted_canvas_history_len[slot_idx] = 0 self.accepted_canvas_history_len[slot_idx].fill_(0)
self.self_conditioning_embeds[slot_idx] = 0 self.self_conditioning_embeds[slot_idx] = 0
def remove_request(self, slot_idx: int) -> None: def remove_request(self, slot_idx: int) -> None:
self.is_encoder_phase[slot_idx] = False self.is_encoder_phase[slot_idx].fill_(False)
self.accepted_canvas_history_len[slot_idx] = 0 self.accepted_canvas_history_len[slot_idx].fill_(0)
self.self_conditioning_embeds[slot_idx] = 0 self.self_conditioning_embeds[slot_idx] = 0
@@ -1148,11 +1152,15 @@ class DiffusionSampler:
ps = input_batch.idx_mapping_np[prefill_indices_np[done_prefill_np]] ps = input_batch.idx_mapping_np[prefill_indices_np[done_prefill_np]]
if len(ps) == 0: if len(ps) == 0:
return return
states.init_canvas(ps) # Move the slot indices across once, up front: indexing a device
self.req_states.draft_tokens[ps, : self.canvas_length] = states.canvas[ps] # tensor with a numpy array copies them over synchronously each time.
ps_gpu = async_copy_to_gpu( ps_gpu = async_copy_to_gpu(
ps.astype(np.int64), device=states.is_encoder_phase.device ps.astype(np.int64), device=states.is_encoder_phase.device
) )
states.init_canvas(ps_gpu)
self.req_states.draft_tokens[ps_gpu, : self.canvas_length] = states.canvas[
ps_gpu
]
states.is_encoder_phase.index_fill_(0, ps_gpu, False) states.is_encoder_phase.index_fill_(0, ps_gpu, False)
def _handle_prefill( def _handle_prefill(
+15 -1
View File
@@ -69,6 +69,7 @@ from vllm.multimodal.processing.processor import (
) )
from vllm.sequence import IntermediateTensors from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from .interfaces import ( from .interfaces import (
MultiModalEmbeddings, MultiModalEmbeddings,
@@ -2039,7 +2040,20 @@ class Gemma4ForConditionalGeneration(
) -> torch.Tensor | None: ) -> torch.Tensor | None:
logits = self.language_model.compute_logits(hidden_states) logits = self.language_model.compute_logits(hidden_states)
if logits is not None and self._suppress_token_ids: if logits is not None and self._suppress_token_ids:
logits[:, self._suppress_token_ids] = -float("inf") # Cache a per-device index tensor for the (static) suppressed-token
# set and use `index_fill_`, so neither the Python-list indices nor
# the scalar fill value take a host roundtrip per call.
cache = getattr(self, "_suppress_token_ids_cache", None)
if cache is None:
cache = {}
self._suppress_token_ids_cache = cache
suppress_idx = cache.get(logits.device)
if suppress_idx is None:
suppress_idx = async_tensor_h2d(
self._suppress_token_ids, dtype=torch.long, device=logits.device
)
cache[logits.device] = suppress_idx
logits.index_fill_(1, suppress_idx, -float("inf"))
return logits return logits
# ------------------------------------------------------------------ # # ------------------------------------------------------------------ #
+2 -2
View File
@@ -548,9 +548,9 @@ class Glm4vVisionEmbeddings(nn.Module):
else: else:
# Convert inputs to tensors if needed # Convert inputs to tensors if needed
if isinstance(lengths, list): if isinstance(lengths, list):
lengths = torch.tensor(lengths, device=device, dtype=torch.long) lengths = async_tensor_h2d(lengths, device=device, dtype=torch.long)
if not isinstance(image_shapes, torch.Tensor): if not isinstance(image_shapes, torch.Tensor):
image_shapes = torch.tensor( image_shapes = async_tensor_h2d(
image_shapes, device=device, dtype=torch.long image_shapes, device=device, dtype=torch.long
) )
+6 -2
View File
@@ -426,12 +426,16 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
), ),
image_num_patches=MultiModalFieldConfig.batched("image"), image_num_patches=MultiModalFieldConfig.batched("image"),
image_embeds=MultiModalFieldConfig.batched("image"), image_embeds=MultiModalFieldConfig.batched("image"),
image_token_id=MultiModalFieldConfig.shared("image", num_images), image_token_id=MultiModalFieldConfig.shared(
"image", num_images, keep_on_cpu=True
),
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes( pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
"video", video_num_patches "video", video_num_patches
), ),
video_num_patches=MultiModalFieldConfig.batched("video"), video_num_patches=MultiModalFieldConfig.batched("video"),
video_token_id=MultiModalFieldConfig.shared("video", num_videos), video_token_id=MultiModalFieldConfig.shared(
"video", num_videos, keep_on_cpu=True
),
) )
def _get_prompt_updates( def _get_prompt_updates(
+3 -1
View File
@@ -478,7 +478,9 @@ class InternVLMultiModalProcessor(
"video", video_num_patches "video", video_num_patches
), ),
video_num_patches=MultiModalFieldConfig.batched("video", keep_on_cpu=True), video_num_patches=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
video_token_id=MultiModalFieldConfig.shared("video", num_videos), video_token_id=MultiModalFieldConfig.shared(
"video", num_videos, keep_on_cpu=True
),
) )
def _get_mm_fields_config( def _get_mm_fields_config(
+5 -4
View File
@@ -871,12 +871,12 @@ def _keye_field_config(
return dict( return dict(
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes), pixel_values=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes), image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
image_grid_thw=MultiModalFieldConfig.batched("image"), image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes( pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
"video", video_grid_sizes "video", video_grid_sizes
), ),
video_embeds=MultiModalFieldConfig.flat_from_sizes("video", video_grid_sizes), video_embeds=MultiModalFieldConfig.flat_from_sizes("video", video_grid_sizes),
video_grid_thw=MultiModalFieldConfig.batched("video"), video_grid_thw=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
) )
@@ -1291,8 +1291,9 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
image_grid_thw = image_input["image_grid_thw"] image_grid_thw = image_input["image_grid_thw"]
assert image_grid_thw.ndim == 2 assert image_grid_thw.ndim == 2
for idx, thaw in enumerate(image_grid_thw): image_grid_thw_list = image_grid_thw.tolist()
thw_tuple = tuple(thaw.detach().cpu().numpy().tolist()) for idx, thw in enumerate(image_grid_thw_list):
thw_tuple = tuple(thw)
numel = np.prod(thw_tuple) numel = np.prod(thw_tuple)
image_grid_hws.append(thw_tuple) image_grid_hws.append(thw_tuple)
image_position_ids = torch.arange(numel) % np.prod(thw_tuple[1:]) image_position_ids = torch.arange(numel) % np.prod(thw_tuple[1:])
+3 -2
View File
@@ -266,9 +266,10 @@ class Phi3HDImageEmbedding(nn.Module):
) )
batch_image_features_proj = [] batch_image_features_proj = []
image_sizes_list = image_sizes.tolist()
# need a for loop to process each image because of different image sizes # need a for loop to process each image because of different image sizes
# (patch arrangement is different for each image) # (patch arrangement is different for each image)
for i, img_size in enumerate(image_sizes): for i, img_size in enumerate(image_sizes_list):
h, w = img_size h, w = img_size
h_crop = h // 336 h_crop = h // 336
w_crop = w // 336 w_crop = w // 336
@@ -426,7 +427,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
) -> Mapping[str, MultiModalFieldConfig]: ) -> Mapping[str, MultiModalFieldConfig]:
return dict( return dict(
pixel_values=MultiModalFieldConfig.batched("image"), pixel_values=MultiModalFieldConfig.batched("image"),
image_sizes=MultiModalFieldConfig.batched("image"), image_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
image_embeds=MultiModalFieldConfig.batched("image"), image_embeds=MultiModalFieldConfig.batched("image"),
) )
+1 -2
View File
@@ -270,8 +270,7 @@ class Phi4MMImageEncoder(nn.Module):
pixel_values = pixel_values.flatten(0, 1) pixel_values = pixel_values.flatten(0, 1)
img_features = self.get_img_features( img_features = self.get_img_features(
pixel_values, pixel_values, image_attention_mask.bool().flatten(0, 1)
image_attention_mask.type(torch.BoolTensor).flatten(0, 1).to(target_device),
) )
base_feat_height_target = self.base_feat_height_target base_feat_height_target = self.base_feat_height_target
+3 -2
View File
@@ -200,7 +200,7 @@ class Step3VLMultiModalProcessor(BaseMultiModalProcessor[Step3VLProcessingInfo])
patch_pixel_values=MultiModalFieldConfig.flat_from_sizes( patch_pixel_values=MultiModalFieldConfig.flat_from_sizes(
"image", num_patches "image", num_patches
), ),
num_patches=MultiModalFieldConfig.batched("image"), num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
patch_newline_mask=MultiModalFieldConfig.flat_from_sizes( patch_newline_mask=MultiModalFieldConfig.flat_from_sizes(
"image", num_patches "image", num_patches
), ),
@@ -682,7 +682,8 @@ class Step3VLForConditionalGeneration(
merged_image_features = [] merged_image_features = []
cur_patch_idx = 0 cur_patch_idx = 0
for i, num_patch in enumerate(num_patches): num_patches_list = num_patches.tolist()
for i, num_patch in enumerate(num_patches_list):
cur_feature = [] cur_feature = []
if num_patch > 0: if num_patch > 0:
patch_slice = patch_image_features[ patch_slice = patch_image_features[
+2
View File
@@ -442,6 +442,7 @@ class CommonAttentionMetadata:
# Needed by FastPrefillAttentionBuilder # Needed by FastPrefillAttentionBuilder
logits_indices_padded: torch.Tensor | None = None logits_indices_padded: torch.Tensor | None = None
num_logits_indices: int | None = None num_logits_indices: int | None = None
max_logits_per_req: int | None = None
# Needed by CrossAttentionBuilder # Needed by CrossAttentionBuilder
encoder_seq_lens: torch.Tensor | None = None encoder_seq_lens: torch.Tensor | None = None
@@ -590,6 +591,7 @@ class CommonAttentionMetadata:
else self.causal, else self.causal,
logits_indices_padded=self.logits_indices_padded, logits_indices_padded=self.logits_indices_padded,
num_logits_indices=self.num_logits_indices, num_logits_indices=self.num_logits_indices,
max_logits_per_req=self.max_logits_per_req,
encoder_seq_lens=maybe_slice_reqs(self.encoder_seq_lens), encoder_seq_lens=maybe_slice_reqs(self.encoder_seq_lens),
encoder_seq_lens_cpu=maybe_slice_reqs(self.encoder_seq_lens_cpu), encoder_seq_lens_cpu=maybe_slice_reqs(self.encoder_seq_lens_cpu),
dcp_local_seq_lens=maybe_slice_reqs(self.dcp_local_seq_lens), dcp_local_seq_lens=maybe_slice_reqs(self.dcp_local_seq_lens),
+9 -5
View File
@@ -150,11 +150,15 @@ class Mamba2AttentionMetadataBuilder(
# Compute seq_idx for prefill only # Compute seq_idx for prefill only
if common.num_prefills > 0: if common.num_prefills > 0:
prep_initial_states = ( prep_initial_states = False
torch.any(common.has_initial_states_p).item() if common.has_initial_states_p is not None:
if common.has_initial_states_p is not None # Same condition as `has_initial_states_p`, but derived from CPU
else False # data so it needs no D2H. `seq_lens_cpu_upper_bound` is precise
) # for prefill rows, which is all this slice covers.
num_computed_tokens_p_cpu, _ = self._prefill_cpu_metadata(
common, common_attn_metadata
)
prep_initial_states = bool((num_computed_tokens_p_cpu > 0).any())
cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p = ( cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p = (
self._build_chunk_metadata_tensors( self._build_chunk_metadata_tensors(
+29 -15
View File
@@ -324,6 +324,33 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
return cu_chunk_seqlen, seq_idx, last_chunk_indices return cu_chunk_seqlen, seq_idx, last_chunk_indices
def _prefill_cpu_metadata(
self,
common: M,
common_attn_metadata: CommonAttentionMetadata,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Prefill context lengths and query offsets, from CPU data only.
`seq_lens_cpu_upper_bound` is precise for prefill rows in all modes
(including async spec decode), so this avoids the D2H sync that
`compute_num_computed_tokens().cpu()` would force.
Returns (num_computed_tokens_p_cpu, query_start_loc_p_cpu).
"""
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
assert seq_lens_cpu is not None
num_reqs = common.num_reqs
num_prefills = common.num_prefills
query_start_loc_p_cpu = (
common_attn_metadata.query_start_loc_cpu[-num_prefills - 1 :]
- common.num_decode_tokens
)
prefill_query_lens_cpu = query_start_loc_p_cpu[1:] - query_start_loc_p_cpu[:-1]
num_computed_tokens_p_cpu = (
seq_lens_cpu[num_reqs - num_prefills : num_reqs] - prefill_query_lens_cpu
)
return num_computed_tokens_p_cpu, query_start_loc_p_cpu
def _build_chunk_metadata_tensors( def _build_chunk_metadata_tensors(
self, self,
chunk_size: int, chunk_size: int,
@@ -334,23 +361,10 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
Compute chunk metadata and return as device tensors. Compute chunk metadata and return as device tensors.
Returns (cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p). Returns (cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p).
""" """
num_reqs = common.num_reqs
num_prefills = common.num_prefills num_prefills = common.num_prefills
num_decode_tokens = common.num_decode_tokens
# Derive prefill context lengths from CPU data only. num_computed_tokens_p_cpu, query_start_loc_p_cpu = self._prefill_cpu_metadata(
# `seq_lens_cpu_upper_bound` is precise for prefill rows in all modes common, common_attn_metadata
# (including async spec decode), so this avoids the D2H sync that
# `compute_num_computed_tokens().cpu()` would force.
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
assert seq_lens_cpu is not None
query_start_loc_p_cpu = (
common_attn_metadata.query_start_loc_cpu[-num_prefills - 1 :]
- num_decode_tokens
)
prefill_query_lens_cpu = query_start_loc_p_cpu[1:] - query_start_loc_p_cpu[:-1]
num_computed_tokens_p_cpu = (
seq_lens_cpu[num_reqs - num_prefills : num_reqs] - prefill_query_lens_cpu
) )
cu_chunk_seqlen, seq_idx, last_chunk_indices = self._compute_chunk_metadata( cu_chunk_seqlen, seq_idx, last_chunk_indices = self._compute_chunk_metadata(
+8 -2
View File
@@ -539,8 +539,14 @@ def make_kv_sharing_fast_prefill_common_attn_metadata(
decode_query_start_loc[:1].fill_(0) # Avoid sync from scalar assignment. decode_query_start_loc[:1].fill_(0) # Avoid sync from scalar assignment.
decode_query_start_loc[1:] = torch.cumsum(num_decode_tokens, dim=0) decode_query_start_loc[1:] = torch.cumsum(num_decode_tokens, dim=0)
decode_max_query_len = int(num_decode_tokens.max().item())
total_num_decode_tokens = int(num_decode_tokens.sum().item()) # `num_decode_tokens` is a histogram over `logits_indices`, so its total is
# just how many there were -- already known as a Python int.
total_num_decode_tokens = num_logits_indices
# Largest per-request logits count.
decode_max_query_len = common_attn_metadata.max_logits_per_req
assert decode_max_query_len is not None
common_attn_metadata = CommonAttentionMetadata( common_attn_metadata = CommonAttentionMetadata(
query_start_loc=decode_query_start_loc, query_start_loc=decode_query_start_loc,
+8 -4
View File
@@ -2009,10 +2009,11 @@ class GPUModelRunner(
) -> tuple[ ) -> tuple[
torch.Tensor, torch.Tensor,
SpecDecodeMetadata | None, SpecDecodeMetadata | None,
int,
]: ]:
""" """
Returns: Returns:
tuple[logits_indices, spec_decode_metadata] tuple[logits_indices, spec_decode_metadata, max_num_sampled_tokens]
""" """
total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
assert total_num_scheduled_tokens > 0 assert total_num_scheduled_tokens > 0
@@ -2324,6 +2325,7 @@ class GPUModelRunner(
return ( return (
logits_indices, logits_indices,
spec_decode_metadata, spec_decode_metadata,
int(num_sampled_tokens.max()),
) )
def _build_attention_metadata( def _build_attention_metadata(
@@ -2335,6 +2337,7 @@ class GPUModelRunner(
num_reqs_padded: int | None = None, num_reqs_padded: int | None = None,
ubatch_slices: UBatchSlices | None = None, ubatch_slices: UBatchSlices | None = None,
logits_indices: torch.Tensor | None = None, logits_indices: torch.Tensor | None = None,
max_num_sampled_tokens: int | None = None,
use_spec_decode: bool = False, use_spec_decode: bool = False,
for_cudagraph_capture: bool = False, for_cudagraph_capture: bool = False,
num_scheduled_tokens: dict[str, int] | None = None, num_scheduled_tokens: dict[str, int] | None = None,
@@ -2509,6 +2512,7 @@ class GPUModelRunner(
if logits_indices is not None and self.cache_config.kv_sharing_fast_prefill: if logits_indices is not None and self.cache_config.kv_sharing_fast_prefill:
cm_base.num_logits_indices = logits_indices.size(0) cm_base.num_logits_indices = logits_indices.size(0)
cm_base.max_logits_per_req = max_num_sampled_tokens
cm_base.logits_indices_padded = self._prepare_kv_sharing_fast_prefill( cm_base.logits_indices_padded = self._prepare_kv_sharing_fast_prefill(
logits_indices logits_indices
) )
@@ -4341,9 +4345,8 @@ class GPUModelRunner(
max_num_scheduled_tokens = int(num_scheduled_tokens_np.max()) max_num_scheduled_tokens = int(num_scheduled_tokens_np.max())
num_tokens_unpadded = scheduler_output.total_num_scheduled_tokens num_tokens_unpadded = scheduler_output.total_num_scheduled_tokens
logits_indices, spec_decode_metadata = self._prepare_inputs( logits_indices, spec_decode_metadata, max_num_sampled_tokens = (
scheduler_output, self._prepare_inputs(scheduler_output, num_scheduled_tokens_np)
num_scheduled_tokens_np,
) )
cascade_attn_prefix_lens = None cascade_attn_prefix_lens = None
@@ -4481,6 +4484,7 @@ class GPUModelRunner(
max_query_len=max_num_scheduled_tokens, max_query_len=max_num_scheduled_tokens,
ubatch_slices=ubatch_slices_attn, ubatch_slices=ubatch_slices_attn,
logits_indices=logits_indices, logits_indices=logits_indices,
max_num_sampled_tokens=max_num_sampled_tokens,
use_spec_decode=use_spec_decode, use_spec_decode=use_spec_decode,
num_scheduled_tokens=scheduler_output.num_scheduled_tokens, num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
cascade_attn_prefix_lens=cascade_attn_prefix_lens, cascade_attn_prefix_lens=cascade_attn_prefix_lens,