[Misc][Refactor] Introduce ExecuteModelData (#4540)
This commit is contained in:
+31
-1
@@ -1,7 +1,7 @@
|
||||
"""Sequence and its related classes."""
|
||||
import copy
|
||||
import enum
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional, Union
|
||||
|
||||
from vllm.block import LogicalTokenBlock
|
||||
@@ -734,3 +734,33 @@ class SamplerOutput:
|
||||
f"sampled_token_probs={sampled_token_probs_repr}, "
|
||||
f"sampled_token_ids={sampled_token_ids_repr}, "
|
||||
f"spec_decode_worker_metrics={self.spec_decode_worker_metrics})")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExecuteModelRequest:
|
||||
"""The model execution request."""
|
||||
# The sequence group metadata list.
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata]
|
||||
# Blocks to swap in. Dict of CPU -> GPU block number.
|
||||
blocks_to_swap_in: Dict[int, int] = field(default_factory=dict)
|
||||
# Blocks to swap out. Dict of GPU -> CPU block number.
|
||||
blocks_to_swap_out: Dict[int, int] = field(default_factory=dict)
|
||||
# Blocks to copy. Source to a list of dest blocks.
|
||||
blocks_to_copy: Dict[int, List[int]] = field(default_factory=dict)
|
||||
# The number of slots for lookahead decoding.
|
||||
num_lookahead_slots: int = 0
|
||||
# The number of requests in the running queue.
|
||||
running_queue_size: int = 0
|
||||
|
||||
def clone(
|
||||
self, seq_group_metadata_list: List[SequenceGroupMetadata]
|
||||
) -> "ExecuteModelRequest":
|
||||
"""Clone the request with a new sequence group metadata list."""
|
||||
return ExecuteModelRequest(
|
||||
seq_group_metadata_list=seq_group_metadata_list,
|
||||
blocks_to_swap_in=self.blocks_to_swap_in.copy(),
|
||||
blocks_to_swap_out=self.blocks_to_swap_out.copy(),
|
||||
blocks_to_copy=self.blocks_to_copy.copy(),
|
||||
num_lookahead_slots=self.num_lookahead_slots,
|
||||
running_queue_size=self.running_queue_size,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user