[ Frontend ] Multiprocessing for OpenAI Server with zeromq (#6883)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com> Co-authored-by: Joe Runde <Joseph.Runde@ibm.com> Co-authored-by: Joe Runde <joe@joerun.de> Co-authored-by: Nick Hill <nickhill@us.ibm.com> Co-authored-by: Simon Mo <simon.mo@hey.com>
This commit is contained in:
co-authored by
Joe Runde
Joe Runde
Nick Hill
Simon Mo
parent
708989341e
commit
ed812a73fa
@@ -1,4 +1,4 @@
|
||||
from functools import lru_cache
|
||||
from functools import lru_cache, partial
|
||||
from typing import Dict, FrozenSet, Iterable, List, Optional, Union
|
||||
|
||||
import torch
|
||||
@@ -40,6 +40,14 @@ def _get_allowed_token_ids_logits_processor(
|
||||
return AllowedTokenIdsLogitsProcessor(allowed_token_ids)
|
||||
|
||||
|
||||
def logit_bias_logits_processor(logit_bias: Dict[str,
|
||||
float], token_ids: List[int],
|
||||
logits: torch.Tensor) -> torch.Tensor:
|
||||
for token_id, bias in logit_bias.items():
|
||||
logits[token_id] += bias
|
||||
return logits
|
||||
|
||||
|
||||
def get_logits_processors(
|
||||
logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]],
|
||||
allowed_token_ids: Optional[List[int]],
|
||||
@@ -64,13 +72,8 @@ def get_logits_processors(
|
||||
raise ValueError("token_id in logit_bias contains "
|
||||
"out-of-vocab token id")
|
||||
|
||||
def logit_bias_logits_processor(token_ids: List[int],
|
||||
logits: torch.Tensor) -> torch.Tensor:
|
||||
for token_id, bias in clamped_logit_bias.items():
|
||||
logits[token_id] += bias
|
||||
return logits
|
||||
|
||||
logits_processors.append(logit_bias_logits_processor)
|
||||
logits_processors.append(
|
||||
partial(logit_bias_logits_processor, clamped_logit_bias))
|
||||
|
||||
if allowed_token_ids is not None:
|
||||
logits_processors.append(
|
||||
|
||||
Reference in New Issue
Block a user