[Misc] Use dataclass for InputMetadata (#3452)
Co-authored-by: youkaichao <youkaichao@126.com>
This commit is contained in:
co-authored by
youkaichao
parent
6b78837b29
commit
abfc4f3387
+10
-27
@@ -1,4 +1,5 @@
|
||||
import contextlib
|
||||
import dataclasses
|
||||
import time
|
||||
from typing import Dict, List, Optional, Tuple, Set, Union
|
||||
|
||||
@@ -521,45 +522,27 @@ class ModelRunner:
|
||||
metadata_dict = {
|
||||
"input_tokens": input_tokens,
|
||||
"input_positions": input_positions,
|
||||
"is_prompt": input_metadata.is_prompt,
|
||||
"slot_mapping": input_metadata.slot_mapping,
|
||||
"prompt_lens": input_metadata.prompt_lens,
|
||||
"max_seq_len": input_metadata.max_seq_len,
|
||||
"start_loc": input_metadata.start_loc,
|
||||
"max_context_len": input_metadata.max_context_len,
|
||||
"context_lens": input_metadata.context_lens,
|
||||
"block_tables": input_metadata.block_tables,
|
||||
"use_cuda_graph": input_metadata.use_cuda_graph,
|
||||
"kv_cache_dtype": input_metadata.kv_cache_dtype,
|
||||
"selected_token_indices":
|
||||
sampling_metadata.selected_token_indices,
|
||||
"lora_requests": lora_requests,
|
||||
"lora_mapping": lora_mapping,
|
||||
}
|
||||
metadata_dict.update(dataclasses.asdict(input_metadata))
|
||||
broadcast_tensor_dict(metadata_dict, src=0)
|
||||
else:
|
||||
metadata_dict = broadcast_tensor_dict(src=0)
|
||||
input_tokens = metadata_dict["input_tokens"]
|
||||
input_positions = metadata_dict["input_positions"]
|
||||
lora_mapping = metadata_dict["lora_mapping"]
|
||||
lora_requests = metadata_dict["lora_requests"]
|
||||
input_metadata = InputMetadata(
|
||||
is_prompt=metadata_dict["is_prompt"],
|
||||
slot_mapping=metadata_dict["slot_mapping"],
|
||||
prompt_lens=metadata_dict["prompt_lens"],
|
||||
max_seq_len=metadata_dict["max_seq_len"],
|
||||
start_loc=metadata_dict["start_loc"],
|
||||
max_context_len=metadata_dict["max_context_len"],
|
||||
context_lens=metadata_dict["context_lens"],
|
||||
block_tables=metadata_dict["block_tables"],
|
||||
use_cuda_graph=metadata_dict["use_cuda_graph"],
|
||||
kv_cache_dtype=metadata_dict["kv_cache_dtype"],
|
||||
)
|
||||
input_tokens = metadata_dict.pop("input_tokens")
|
||||
input_positions = metadata_dict.pop("input_positions")
|
||||
selected_token_indices = metadata_dict.pop(
|
||||
"selected_token_indices")
|
||||
lora_mapping = metadata_dict.pop("lora_mapping")
|
||||
lora_requests = metadata_dict.pop("lora_requests")
|
||||
input_metadata = InputMetadata(**metadata_dict)
|
||||
sampling_metadata = SamplingMetadata(
|
||||
seq_groups=None,
|
||||
seq_data=None,
|
||||
prompt_lens=None,
|
||||
selected_token_indices=metadata_dict["selected_token_indices"],
|
||||
selected_token_indices=selected_token_indices,
|
||||
categorized_sample_indices=None,
|
||||
generators=None,
|
||||
perform_sampling=False,
|
||||
|
||||
Reference in New Issue
Block a user