[Performance] Split FlashAttn attention and cache update (#25954)
Signed-off-by: ElizaWszola <ewszola@redhat.com> Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Luka Govedič <luka.govedic@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Signed-off-by: Luka Govedič <lgovedic@redhat.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: Varun Sundar Rabindranath <varunsundar08@gmail.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: Luka Govedič <luka.govedic@gmail.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Luka Govedič <lgovedic@redhat.com>
This commit is contained in:
co-authored by
mgoin
Varun Sundar Rabindranath
Matthew Bonanni
Luka Govedič
Luka Govedič
Lucas Wilkinson
Luka Govedič
parent
0118cdcc02
commit
a28b94e6ef
@@ -140,6 +140,18 @@ def init_kv_cache(
|
||||
return kv_caches
|
||||
|
||||
|
||||
def build_slot_mappings_by_layer(
|
||||
slot_mappings: torch.Tensor,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
slot_mappings_by_layer: dict[str, torch.Tensor] = {}
|
||||
for i, kv_cache_group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
slot_mapping = slot_mappings[i]
|
||||
for layer_name in kv_cache_group.layer_names:
|
||||
slot_mappings_by_layer[layer_name] = slot_mapping
|
||||
return slot_mappings_by_layer
|
||||
|
||||
|
||||
def build_attn_metadata(
|
||||
attn_metadata_builders: list[AttentionMetadataBuilder],
|
||||
num_reqs: int,
|
||||
|
||||
@@ -14,7 +14,10 @@ from vllm.distributed.parallel_state import graph_capture, is_global_first_rank
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.v1.attention.backend import AttentionMetadataBuilder
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
|
||||
from vllm.v1.worker.gpu.attn_utils import (
|
||||
build_attn_metadata,
|
||||
build_slot_mappings_by_layer,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.dp_utils import make_num_tokens_across_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBuffers
|
||||
@@ -88,7 +91,7 @@ class CudaGraphManager:
|
||||
positions = mrope_positions[:, :num_tokens]
|
||||
if inputs_embeds is not None:
|
||||
inputs_embeds = inputs_embeds[:num_tokens]
|
||||
attn_metadata = prepare_inputs_to_capture(
|
||||
attn_metadata, slot_mappings = prepare_inputs_to_capture(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
input_buffers,
|
||||
@@ -98,6 +101,9 @@ class CudaGraphManager:
|
||||
kv_cache_config,
|
||||
)
|
||||
num_tokens_across_dp = make_num_tokens_across_dp(self.dp_size, num_tokens)
|
||||
slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
slot_mappings, kv_cache_config
|
||||
)
|
||||
|
||||
# Warm up.
|
||||
with set_forward_context(
|
||||
@@ -106,6 +112,7 @@ class CudaGraphManager:
|
||||
num_tokens=num_tokens,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
slot_mapping=slot_mappings_by_layer,
|
||||
):
|
||||
hidden_states = model(
|
||||
input_ids=input_ids,
|
||||
@@ -125,6 +132,7 @@ class CudaGraphManager:
|
||||
num_tokens=num_tokens,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
slot_mapping=slot_mappings_by_layer,
|
||||
),
|
||||
torch.cuda.graph(graph, self.pool),
|
||||
):
|
||||
@@ -244,7 +252,7 @@ def prepare_inputs_to_capture(
|
||||
attn_metadata_builders: list[AttentionMetadataBuilder],
|
||||
max_model_len: int,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> dict[str, Any]:
|
||||
) -> tuple[dict[str, Any], torch.Tensor]:
|
||||
num_tokens_per_req = num_tokens // num_reqs
|
||||
|
||||
query_start_loc_np = np.arange(num_reqs + 1, dtype=np.int32) * num_tokens_per_req
|
||||
@@ -274,4 +282,4 @@ def prepare_inputs_to_capture(
|
||||
slot_mappings=slot_mappings,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
return attn_metadata
|
||||
return attn_metadata, slot_mappings
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.worker.gpu.async_utils import AsyncOutput
|
||||
from vllm.v1.worker.gpu.attn_utils import (
|
||||
build_attn_metadata,
|
||||
build_slot_mappings_by_layer,
|
||||
get_kv_cache_spec,
|
||||
init_attn_backend,
|
||||
init_kv_cache,
|
||||
@@ -881,6 +882,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if self.uses_mrope:
|
||||
assert input_batch.mrope_positions is not None
|
||||
positions = input_batch.mrope_positions
|
||||
slot_mappings = self.block_tables.compute_slot_mappings(
|
||||
input_batch.idx_mapping,
|
||||
input_batch.query_start_loc,
|
||||
input_batch.positions[: input_batch.num_tokens],
|
||||
)
|
||||
slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
slot_mappings, self.kv_cache_config
|
||||
)
|
||||
with set_forward_context(
|
||||
input_batch.attn_metadata,
|
||||
self.vllm_config,
|
||||
@@ -888,6 +897,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# TODO(woosuk): Support piecewise CUDA graph.
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
slot_mapping=slot_mappings_by_layer,
|
||||
):
|
||||
self.kv_connector.pre_forward(scheduler_output)
|
||||
hidden_states = self.model(
|
||||
|
||||
@@ -314,6 +314,7 @@ class ExecuteModelState(NamedTuple):
|
||||
aux_hidden_states: list[torch.Tensor] | None
|
||||
ec_connector_output: ECConnectorOutput | None
|
||||
cudagraph_stats: CUDAGraphStat | None
|
||||
slot_mappings: dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None
|
||||
|
||||
|
||||
class GPUModelRunner(
|
||||
@@ -1595,6 +1596,7 @@ class GPUModelRunner(
|
||||
for_cudagraph_capture: bool = False,
|
||||
num_scheduled_tokens: dict[str, int] | None = None,
|
||||
cascade_attn_prefix_lens: list[list[int]] | None = None,
|
||||
slot_mappings: dict[int, torch.Tensor] | None = None,
|
||||
) -> tuple[PerLayerAttnMetadata, CommonAttentionMetadata | None]:
|
||||
"""
|
||||
:return: tuple[attn_metadata, spec_decode_common_attn_metadata]
|
||||
@@ -1628,7 +1630,7 @@ class GPUModelRunner(
|
||||
|
||||
kv_cache_groups = self.kv_cache_config.kv_cache_groups
|
||||
|
||||
def _get_block_table_and_slot_mapping(kv_cache_gid: int):
|
||||
def _get_block_table(kv_cache_gid: int):
|
||||
assert num_reqs_padded is not None and num_tokens_padded is not None
|
||||
kv_cache_spec = kv_cache_groups[kv_cache_gid].kv_cache_spec
|
||||
if isinstance(kv_cache_spec, EncoderOnlyAttentionSpec):
|
||||
@@ -1637,24 +1639,19 @@ class GPUModelRunner(
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
slot_mapping = torch.zeros(
|
||||
(num_tokens_padded,),
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
else:
|
||||
blk_table = self.input_batch.block_table[kv_cache_gid]
|
||||
blk_table_tensor = blk_table.get_device_tensor(num_reqs_padded)
|
||||
slot_mapping = blk_table.slot_mapping.gpu[:num_tokens_padded]
|
||||
|
||||
# Fill unused with -1. Needed for reshape_and_cache in full cuda
|
||||
# graph mode. `blk_table_tensor` -1 to match mamba PAD_SLOT_ID
|
||||
slot_mapping[num_tokens:num_tokens_padded].fill_(-1)
|
||||
blk_table_tensor[num_reqs:num_reqs_padded].fill_(-1)
|
||||
return blk_table_tensor
|
||||
|
||||
return blk_table_tensor, slot_mapping
|
||||
assert slot_mappings is not None
|
||||
block_table_gid_0 = _get_block_table(0)
|
||||
slot_mapping_gid_0 = slot_mappings[0]
|
||||
|
||||
block_table_gid_0, slot_mapping_gid_0 = _get_block_table_and_slot_mapping(0)
|
||||
if self.model_config.enable_return_routed_experts:
|
||||
self.slot_mapping = slot_mapping_gid_0[:num_tokens].cpu().numpy()
|
||||
cm_base = CommonAttentionMetadata(
|
||||
@@ -1779,9 +1776,8 @@ class GPUModelRunner(
|
||||
num_reqs_padded,
|
||||
)
|
||||
if kv_cache_gid > 0:
|
||||
cm.block_table_tensor, cm.slot_mapping = (
|
||||
_get_block_table_and_slot_mapping(kv_cache_gid)
|
||||
)
|
||||
cm.block_table_tensor = _get_block_table(kv_cache_gid)
|
||||
cm.slot_mapping = slot_mappings[kv_cache_gid]
|
||||
|
||||
if self.speculative_config and spec_decode_common_attn_metadata is None:
|
||||
if isinstance(self.drafter, EagleProposer):
|
||||
@@ -3119,6 +3115,80 @@ class GPUModelRunner(
|
||||
pyt_hooks.register_hooks(self.model, self.model.__class__.__name__)
|
||||
self.layerwise_nvtx_hooks_registered = True
|
||||
|
||||
def _get_slot_mappings(
|
||||
self,
|
||||
num_tokens_padded: int,
|
||||
num_reqs_padded: int,
|
||||
num_tokens_unpadded: int,
|
||||
ubatch_slices: "UBatchSlices | None" = None,
|
||||
) -> tuple[
|
||||
dict[int, torch.Tensor] | None,
|
||||
dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None,
|
||||
]:
|
||||
"""
|
||||
Build slot mappings in both formats needed by the system.
|
||||
|
||||
Args:
|
||||
num_tokens_padded: Total number of tokens (padded)
|
||||
num_reqs_padded: Total number of requests (padded)
|
||||
num_tokens_unpadded: Actual number of tokens (unpadded)
|
||||
ubatch_slices: Optional ubatch slicing info for DBO
|
||||
|
||||
Returns:
|
||||
A tuple of:
|
||||
- slot_mappings_by_gid: dict[int, torch.Tensor] for attention metadata
|
||||
- slot_mappings_by_layer: dict[str, torch.Tensor] or list for ForwardContext
|
||||
"""
|
||||
if not (
|
||||
hasattr(self, "kv_cache_config")
|
||||
and self.kv_cache_config is not None
|
||||
and len(self.kv_cache_config.kv_cache_groups) > 0
|
||||
):
|
||||
return None, None
|
||||
|
||||
def _get_slot_mapping(kv_cache_gid: int):
|
||||
assert num_reqs_padded is not None and num_tokens_padded is not None
|
||||
kv_cache_spec = self.kv_cache_config.kv_cache_groups[
|
||||
kv_cache_gid
|
||||
].kv_cache_spec
|
||||
if isinstance(kv_cache_spec, EncoderOnlyAttentionSpec):
|
||||
slot_mapping = torch.zeros(
|
||||
(num_tokens_padded,),
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
else:
|
||||
blk_table = self.input_batch.block_table[kv_cache_gid]
|
||||
slot_mapping = blk_table.slot_mapping.gpu[:num_tokens_padded]
|
||||
|
||||
# Fill unused with -1. Needed for reshape_and_cache in full cuda
|
||||
# graph mode. `blk_table_tensor` -1 to match mamba PAD_SLOT_ID
|
||||
slot_mapping[num_tokens_unpadded:num_tokens_padded].fill_(-1)
|
||||
|
||||
return slot_mapping
|
||||
|
||||
slot_mappings_by_gid = {
|
||||
gid: _get_slot_mapping(gid)
|
||||
for gid, _ in enumerate(self.kv_cache_config.kv_cache_groups)
|
||||
}
|
||||
|
||||
slot_mappings_by_layer: dict[str, torch.Tensor] = {}
|
||||
for gid, kv_cache_group in enumerate(self.kv_cache_config.kv_cache_groups):
|
||||
slot_mapping = slot_mappings_by_gid[gid]
|
||||
for layer_name in kv_cache_group.layer_names:
|
||||
slot_mappings_by_layer[layer_name] = slot_mapping
|
||||
|
||||
if ubatch_slices is not None:
|
||||
result: list[dict[str, torch.Tensor]] = []
|
||||
for ubatch in ubatch_slices:
|
||||
sliced_mappings: dict[str, torch.Tensor] = {}
|
||||
for layer_name, slot_mapping in slot_mappings_by_layer.items():
|
||||
sliced_mappings[layer_name] = slot_mapping[ubatch.token_slice]
|
||||
result.append(sliced_mappings)
|
||||
return slot_mappings_by_gid, result
|
||||
|
||||
return slot_mappings_by_gid, slot_mappings_by_layer
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
self,
|
||||
@@ -3248,6 +3318,17 @@ class GPUModelRunner(
|
||||
ubatch_slices_padded,
|
||||
)
|
||||
|
||||
# True if any attention backend handles KV cache update separately
|
||||
# from forward() (i.e., forward_includes_kv_cache_update=False). When true,
|
||||
# slot_mappings must use padded dimensions to match the key/value tensors.
|
||||
has_separate_kv_update = not all(
|
||||
all(
|
||||
g.backend.forward_includes_kv_cache_update
|
||||
for g in self.attn_groups[id]
|
||||
)
|
||||
for id, spec in enumerate(self.kv_cache_config.kv_cache_groups)
|
||||
if not isinstance(spec.kv_cache_spec, EncoderOnlyAttentionSpec)
|
||||
)
|
||||
pad_attn = cudagraph_mode == CUDAGraphMode.FULL
|
||||
|
||||
if self.cache_config.mamba_cache_mode == "align":
|
||||
@@ -3265,6 +3346,17 @@ class GPUModelRunner(
|
||||
use_spec_decode = len(scheduler_output.scheduled_spec_decode_tokens) > 0
|
||||
ubatch_slices_attn = ubatch_slices_padded if pad_attn else ubatch_slices
|
||||
|
||||
slot_mappings_by_group, slot_mappings = self._get_slot_mappings(
|
||||
num_tokens_padded=num_tokens_padded
|
||||
if pad_attn or has_separate_kv_update
|
||||
else num_tokens_unpadded,
|
||||
num_reqs_padded=(
|
||||
num_reqs_padded if pad_attn or has_separate_kv_update else num_reqs
|
||||
),
|
||||
num_tokens_unpadded=num_tokens_unpadded,
|
||||
ubatch_slices=ubatch_slices_padded,
|
||||
)
|
||||
|
||||
attn_metadata, spec_decode_common_attn_metadata = (
|
||||
self._build_attention_metadata(
|
||||
num_tokens=num_tokens_unpadded,
|
||||
@@ -3277,6 +3369,7 @@ class GPUModelRunner(
|
||||
use_spec_decode=use_spec_decode,
|
||||
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
|
||||
cascade_attn_prefix_lens=cascade_attn_prefix_lens,
|
||||
slot_mappings=slot_mappings_by_group,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -3317,6 +3410,7 @@ class GPUModelRunner(
|
||||
cudagraph_runtime_mode=cudagraph_mode,
|
||||
batch_descriptor=batch_desc,
|
||||
ubatch_slices=ubatch_slices_padded,
|
||||
slot_mapping=slot_mappings,
|
||||
skip_compiled=has_encoder_input,
|
||||
),
|
||||
record_function_or_nullcontext("gpu_model_runner: forward"),
|
||||
@@ -3399,6 +3493,7 @@ class GPUModelRunner(
|
||||
aux_hidden_states,
|
||||
ec_connector_output,
|
||||
cudagraph_stats,
|
||||
slot_mappings,
|
||||
)
|
||||
self.kv_connector_output = kv_connector_output
|
||||
return None
|
||||
@@ -3435,6 +3530,7 @@ class GPUModelRunner(
|
||||
aux_hidden_states,
|
||||
ec_connector_output,
|
||||
cudagraph_stats,
|
||||
slot_mappings,
|
||||
) = self.execute_model_state
|
||||
# Clear ephemeral state.
|
||||
self.execute_model_state = None
|
||||
@@ -3468,6 +3564,7 @@ class GPUModelRunner(
|
||||
aux_hidden_states,
|
||||
spec_decode_metadata,
|
||||
spec_decode_common_attn_metadata,
|
||||
slot_mappings,
|
||||
)
|
||||
self._copy_draft_token_ids_to_cpu(scheduler_output)
|
||||
|
||||
@@ -3676,6 +3773,7 @@ class GPUModelRunner(
|
||||
aux_hidden_states: list[torch.Tensor] | None,
|
||||
spec_decode_metadata: SpecDecodeMetadata | None,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
slot_mappings: dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None,
|
||||
) -> list[list[int]] | torch.Tensor:
|
||||
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
spec_config = self.speculative_config
|
||||
@@ -3687,11 +3785,14 @@ class GPUModelRunner(
|
||||
sampled_token_ids,
|
||||
self.input_batch.num_tokens_no_spec,
|
||||
self.input_batch.token_ids_cpu,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
elif spec_config.method == "suffix":
|
||||
assert isinstance(sampled_token_ids, list)
|
||||
assert isinstance(self.drafter, SuffixDecodingProposer)
|
||||
draft_token_ids = self.drafter.propose(self.input_batch, sampled_token_ids)
|
||||
draft_token_ids = self.drafter.propose(
|
||||
self.input_batch, sampled_token_ids, slot_mappings=slot_mappings
|
||||
)
|
||||
elif spec_config.method == "medusa":
|
||||
assert isinstance(sampled_token_ids, list)
|
||||
assert isinstance(self.drafter, MedusaProposer)
|
||||
@@ -3716,6 +3817,7 @@ class GPUModelRunner(
|
||||
draft_token_ids = self.drafter.propose(
|
||||
target_hidden_states=hidden_states,
|
||||
sampling_metadata=sampling_metadata,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
elif spec_config.use_eagle() or spec_config.uses_draft_model():
|
||||
assert isinstance(self.drafter, EagleProposer | DraftModelProposer)
|
||||
@@ -3826,6 +3928,7 @@ class GPUModelRunner(
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
mm_embed_inputs=mm_embed_inputs,
|
||||
num_rejected_tokens_gpu=num_rejected_tokens_gpu,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
|
||||
return draft_token_ids
|
||||
@@ -4406,6 +4509,13 @@ class GPUModelRunner(
|
||||
|
||||
attn_metadata: PerLayerAttnMetadata | None = None
|
||||
|
||||
slot_mappings_by_group, slot_mappings = self._get_slot_mappings(
|
||||
num_tokens_padded=num_tokens,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
num_tokens_unpadded=num_tokens_unpadded,
|
||||
ubatch_slices=ubatch_slices_padded,
|
||||
)
|
||||
|
||||
# If force_attention is True, we always capture attention. Otherwise,
|
||||
# it only happens for cudagraph_runtime_mode=FULL.
|
||||
if force_attention or cudagraph_runtime_mode == CUDAGraphMode.FULL:
|
||||
@@ -4431,6 +4541,7 @@ class GPUModelRunner(
|
||||
max_query_len=max_query_len,
|
||||
ubatch_slices=ubatch_slices_padded if pad_attn else ubatch_slices,
|
||||
for_cudagraph_capture=is_graph_capturing,
|
||||
slot_mappings=slot_mappings_by_group,
|
||||
)
|
||||
|
||||
with self.maybe_dummy_run_with_lora(
|
||||
@@ -4499,6 +4610,7 @@ class GPUModelRunner(
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
batch_descriptor=batch_desc,
|
||||
ubatch_slices=ubatch_slices_padded,
|
||||
slot_mapping=slot_mappings,
|
||||
),
|
||||
):
|
||||
outputs = self.model(
|
||||
@@ -4545,6 +4657,7 @@ class GPUModelRunner(
|
||||
num_tokens,
|
||||
use_cudagraphs=use_cudagraphs,
|
||||
is_graph_capturing=is_graph_capturing,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
|
||||
# We register layerwise NVTX hooks here after the first dynamo tracing is
|
||||
|
||||
@@ -295,6 +295,7 @@ class UBatchWrapper:
|
||||
self,
|
||||
ubatch_slices,
|
||||
attn_metadata,
|
||||
slot_mapping,
|
||||
input_ids,
|
||||
positions,
|
||||
inputs_embeds,
|
||||
@@ -306,6 +307,9 @@ class UBatchWrapper:
|
||||
) -> list[UbatchMetadata]:
|
||||
# Create one forward context per ubatch
|
||||
forward_contexts = []
|
||||
# slot_mapping can be None, an empty dict (from create_forward_context
|
||||
# converting None to {}), or a list of dicts (one per ubatch)
|
||||
has_slot_mapping = slot_mapping and isinstance(slot_mapping, list)
|
||||
for i, ubatch_slice in enumerate(ubatch_slices):
|
||||
forward_contexts.append(
|
||||
create_forward_context(
|
||||
@@ -314,6 +318,7 @@ class UBatchWrapper:
|
||||
dp_metadata=dp_metadata[i],
|
||||
batch_descriptor=batch_descriptor,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
slot_mapping=slot_mapping[i] if has_slot_mapping else None,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -406,6 +411,7 @@ class UBatchWrapper:
|
||||
return self.cudagraph_wrapper(*args, **kwargs)
|
||||
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
num_tokens = (
|
||||
ubatch_slices[0].token_slice.stop - ubatch_slices[0].token_slice.start
|
||||
) * 2
|
||||
@@ -440,6 +446,7 @@ class UBatchWrapper:
|
||||
ubatch_metadata = self._make_ubatch_metadata(
|
||||
ubatch_slices=ubatch_slices,
|
||||
attn_metadata=attn_metadata,
|
||||
slot_mapping=slot_mapping,
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
@@ -462,6 +469,7 @@ class UBatchWrapper:
|
||||
ubatch_metadata = self._make_ubatch_metadata(
|
||||
ubatch_slices=ubatch_slices,
|
||||
attn_metadata=attn_metadata,
|
||||
slot_mapping=slot_mapping,
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
|
||||
Reference in New Issue
Block a user