[Model][1/N] Support multiple poolers at model level (#21227)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -15,3 +15,11 @@ class PoolingMetadata:
|
||||
prompt_lens: torch.Tensor
|
||||
prompt_token_ids: Optional[torch.Tensor]
|
||||
pooling_params: list[PoolingParams]
|
||||
|
||||
def __getitem__(self, indices: slice):
|
||||
return PoolingMetadata(
|
||||
prompt_lens=self.prompt_lens[indices],
|
||||
prompt_token_ids=None if self.prompt_token_ids is None else
|
||||
self.prompt_token_ids[indices],
|
||||
pooling_params=self.pooling_params[indices],
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user