[Perf] Remove redundant device copies for CPU-only pooling token IDs, 48.9% E2E throughput improvement (#38139)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -18,7 +18,7 @@ ActivationFn = Callable[[_T], _T]
|
||||
@dataclass(frozen=True)
|
||||
class PoolingParamsUpdate:
|
||||
requires_token_ids: bool = False
|
||||
"""Set this flag to enable `get_prompt_token_ids` for your pooler."""
|
||||
"""Set this flag to enable prompt token IDs for your pooler."""
|
||||
|
||||
def __or__(self, other: "PoolingParamsUpdate") -> "PoolingParamsUpdate":
|
||||
return PoolingParamsUpdate(
|
||||
|
||||
Reference in New Issue
Block a user