Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
co-authored by
Woosuk Kwon
parent
22aeb43007
commit
edf927bc9f
@@ -101,9 +101,6 @@ class CudaGraphManager:
|
||||
kv_cache_config,
|
||||
)
|
||||
num_tokens_across_dp = make_num_tokens_across_dp(self.dp_size, num_tokens)
|
||||
slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
slot_mappings, kv_cache_config
|
||||
)
|
||||
|
||||
# Warm up.
|
||||
with set_forward_context(
|
||||
@@ -112,7 +109,7 @@ class CudaGraphManager:
|
||||
num_tokens=num_tokens,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
slot_mapping=slot_mappings_by_layer,
|
||||
slot_mapping=slot_mappings,
|
||||
):
|
||||
hidden_states = model(
|
||||
input_ids=input_ids,
|
||||
@@ -132,7 +129,7 @@ class CudaGraphManager:
|
||||
num_tokens=num_tokens,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
slot_mapping=slot_mappings_by_layer,
|
||||
slot_mapping=slot_mappings,
|
||||
),
|
||||
torch.cuda.graph(graph, self.pool),
|
||||
):
|
||||
@@ -252,7 +249,7 @@ def prepare_inputs_to_capture(
|
||||
attn_metadata_builders: list[AttentionMetadataBuilder],
|
||||
max_model_len: int,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> tuple[dict[str, Any], torch.Tensor]:
|
||||
) -> tuple[dict[str, Any], dict[str, torch.Tensor]]:
|
||||
num_tokens_per_req = num_tokens // num_reqs
|
||||
|
||||
query_start_loc_np = np.arange(num_reqs + 1, dtype=np.int32) * num_tokens_per_req
|
||||
@@ -269,6 +266,9 @@ def prepare_inputs_to_capture(
|
||||
|
||||
input_block_tables = [x[:num_reqs] for x in block_tables.input_block_tables]
|
||||
slot_mappings = block_tables.slot_mappings[:, :num_tokens]
|
||||
slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
slot_mappings, kv_cache_config
|
||||
)
|
||||
|
||||
attn_metadata = build_attn_metadata(
|
||||
attn_metadata_builders=attn_metadata_builders,
|
||||
@@ -282,4 +282,4 @@ def prepare_inputs_to_capture(
|
||||
slot_mappings=slot_mappings,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
return attn_metadata, slot_mappings
|
||||
return attn_metadata, slot_mappings_by_layer
|
||||
|
||||
Reference in New Issue
Block a user