[Core] Eliminate parallel worker per-step task scheduling overhead (#4894)
This commit is contained in:
@@ -47,7 +47,7 @@ class EmbeddingModelRunner(ModelRunner):
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
self,
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
|
||||
kv_caches: List[torch.Tensor],
|
||||
) -> Optional[PoolerOutput]:
|
||||
(input_tokens, input_positions, attn_metadata, pooling_metadata,
|
||||
@@ -84,10 +84,11 @@ class EmbeddingModelRunner(ModelRunner):
|
||||
|
||||
def prepare_input_tensors(
|
||||
self,
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, PoolingMetadata,
|
||||
Set[LoRARequest], LoRAMapping, torch.Tensor]:
|
||||
if self.is_driver_worker:
|
||||
assert seq_group_metadata_list is not None
|
||||
# Prepare input tensors.
|
||||
(
|
||||
input_tokens,
|
||||
|
||||
Reference in New Issue
Block a user