[CORE] Prompt Embeddings Support for v1 Engine (#24278)
Signed-off-by: Andrew Sansom <andrew@protopia.ai> Signed-off-by: Andrew Sansom <qthequartermasterman@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
co-authored by
Cyrus Leung
parent
9fac6aa30b
commit
9a4600e4dc
@@ -243,7 +243,7 @@ class AdapterLogitsProcessor(LogitsProcessor):
|
||||
def _new_state(
|
||||
self,
|
||||
params: SamplingParams,
|
||||
prompt_ids: list[int],
|
||||
prompt_ids: Optional[list[int]],
|
||||
output_ids: list[int],
|
||||
) -> Optional[partial[torch.Tensor]]:
|
||||
"""Return state representation for new request
|
||||
|
||||
@@ -187,7 +187,8 @@ class MinTokensLogitsProcessor(LogitsProcessor):
|
||||
|
||||
@staticmethod
|
||||
def add_request(
|
||||
params: SamplingParams, _: list[int], output_tok_ids: list[int]
|
||||
params: SamplingParams, _: Optional[list[int]],
|
||||
output_tok_ids: list[int]
|
||||
) -> Optional[tuple[int, Sequence[int], set[int]]]:
|
||||
min_tokens = params.min_tokens
|
||||
if not min_tokens or len(output_tok_ids) >= min_tokens:
|
||||
@@ -234,7 +235,8 @@ class MinTokensLogitsProcessor(LogitsProcessor):
|
||||
|
||||
def process_dict_updates(
|
||||
req_entries: dict[int, T], batch_update: Optional[BatchUpdate],
|
||||
new_state: Callable[[SamplingParams, list[int], list[int]], Optional[T]]
|
||||
new_state: Callable[[SamplingParams, Optional[list[int]], list[int]],
|
||||
Optional[T]]
|
||||
) -> bool:
|
||||
"""Utility function to update dict state for sparse LogitsProcessors."""
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ RemovedRequest = int
|
||||
|
||||
# (index, params, prompt_tok_ids, output_tok_ids) tuples for new
|
||||
# requests added to the batch.
|
||||
AddedRequest = tuple[int, SamplingParams, list[int], list[int]]
|
||||
AddedRequest = tuple[int, SamplingParams, Optional[list[int]], list[int]]
|
||||
|
||||
# (index 1, index 2, directionality) tuples representing
|
||||
# one-way moves or two-way swaps of requests in batch
|
||||
|
||||
Reference in New Issue
Block a user