[Perf][Frontend] Vectorize Cohere binary embedding bit-packing (#52277)

Signed-off-by: Fangchen Li <fangchen.li@outlook.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Fangchen Li
2026-08-14 09:15:59 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 624999aae5
commit 103c419f04
2 changed files with 57 additions and 24 deletions
+22 -24
View File
@@ -12,6 +12,7 @@ import time
from collections.abc import Sequence
from typing import Annotated, Any, Literal, TypeAlias
import numpy as np
import pybase64 as base64
from pydantic import BaseModel, BeforeValidator, Field, model_validator
@@ -298,31 +299,28 @@ def _pack_binary_embeddings(
) -> list[list[int]]:
"""Bit-pack float embeddings: positive -> 1, negative -> 0.
Each bit is shifted left by ``7 - idx%8``, and every 8 bits are packed
into one byte.
Bits are packed MSB-first, eight per byte.
"""
result: list[list[int]] = []
for embedding in float_embeddings:
dim = len(embedding)
if dim % 8 != 0:
raise ValueError(
"Embedding dimension must be a multiple of 8 for binary "
f"embedding types, but got {dim}."
)
packed_len = dim // 8
packed: list[int] = []
byte_val = 0
for idx, value in enumerate(embedding):
bit = 1 if value >= 0 else 0
byte_val += bit << (7 - idx % 8)
if (idx + 1) % 8 == 0:
if signed:
byte_val -= _UNSIGNED_TO_SIGNED_DIFF
packed.append(byte_val)
byte_val = 0
assert len(packed) == packed_len
result.append(packed)
return result
if not float_embeddings:
return []
array = np.asarray(float_embeddings, dtype=np.float64)
if array.ndim != 2:
raise ValueError(
f"Expected a 2D batch of embeddings, but got {array.ndim}D input."
)
dim = array.shape[1]
if dim % 8 != 0:
raise ValueError(
"Embedding dimension must be a multiple of 8 for binary "
f"embedding types, but got {dim}."
)
packed = np.packbits(array >= 0, axis=-1)
if signed:
packed = packed.astype(np.int16) - _UNSIGNED_TO_SIGNED_DIFF
return packed.tolist()
def _encode_base64_embeddings(