[Perf][Frontend] Vectorize Cohere binary embedding bit-packing (#52277)
Signed-off-by: Fangchen Li <fangchen.li@outlook.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
624999aae5
commit
103c419f04
@@ -75,6 +75,41 @@ class TestBuildTypedEmbeddingsBinary:
|
||||
# 0.0 >= 0 is True, so bit=1 for all => 127 (signed)
|
||||
assert result.binary[0] == [127]
|
||||
|
||||
def test_negative_zero_treated_as_positive(self):
|
||||
"""Packing follows ``value >= 0``, not the IEEE-754 sign bit.
|
||||
|
||||
``-0.0 >= 0`` is True, so these bits must be set. An implementation
|
||||
reading the sign bit instead would pack -128 here.
|
||||
"""
|
||||
result = build_typed_embeddings([[-0.0] * 8], ["binary"])
|
||||
assert result.binary is not None
|
||||
assert result.binary[0] == [127]
|
||||
|
||||
def test_negative_denormal_treated_as_negative(self):
|
||||
"""Sign is decided at float64 precision.
|
||||
|
||||
The smallest float64 denormal underflows to -0.0 in float32, where
|
||||
``>= 0`` is True; narrowing the comparison would wrongly pack 127.
|
||||
"""
|
||||
result = build_typed_embeddings([[-5e-324] * 8], ["binary"])
|
||||
assert result.binary is not None
|
||||
assert result.binary[0] == [-128]
|
||||
|
||||
def test_empty_input(self):
|
||||
result = build_typed_embeddings([], ["binary", "ubinary"])
|
||||
assert result.binary == []
|
||||
assert result.ubinary == []
|
||||
|
||||
@pytest.mark.parametrize("embs", [[0.1] * 8, [[[0.1] * 8] * 2]], ids=["1d", "3d"])
|
||||
def test_non_2d_input_raises(self, embs):
|
||||
"""A malformed batch must raise rather than pack along the last axis.
|
||||
|
||||
Packing a 1D or 3D input would silently emit wrongly shaped
|
||||
``binary``/``ubinary`` values instead of reporting an error.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="2D batch"):
|
||||
build_typed_embeddings(embs, ["binary"])
|
||||
|
||||
def test_non_multiple_of_8_raises(self):
|
||||
embs = [[0.1] * 7]
|
||||
with pytest.raises(ValueError, match="multiple of 8"):
|
||||
|
||||
@@ -12,6 +12,7 @@ import time
|
||||
from collections.abc import Sequence
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
from pydantic import BaseModel, BeforeValidator, Field, model_validator
|
||||
|
||||
@@ -298,31 +299,28 @@ def _pack_binary_embeddings(
|
||||
) -> list[list[int]]:
|
||||
"""Bit-pack float embeddings: positive -> 1, negative -> 0.
|
||||
|
||||
Each bit is shifted left by ``7 - idx%8``, and every 8 bits are packed
|
||||
into one byte.
|
||||
Bits are packed MSB-first, eight per byte.
|
||||
"""
|
||||
result: list[list[int]] = []
|
||||
for embedding in float_embeddings:
|
||||
dim = len(embedding)
|
||||
if dim % 8 != 0:
|
||||
raise ValueError(
|
||||
"Embedding dimension must be a multiple of 8 for binary "
|
||||
f"embedding types, but got {dim}."
|
||||
)
|
||||
packed_len = dim // 8
|
||||
packed: list[int] = []
|
||||
byte_val = 0
|
||||
for idx, value in enumerate(embedding):
|
||||
bit = 1 if value >= 0 else 0
|
||||
byte_val += bit << (7 - idx % 8)
|
||||
if (idx + 1) % 8 == 0:
|
||||
if signed:
|
||||
byte_val -= _UNSIGNED_TO_SIGNED_DIFF
|
||||
packed.append(byte_val)
|
||||
byte_val = 0
|
||||
assert len(packed) == packed_len
|
||||
result.append(packed)
|
||||
return result
|
||||
if not float_embeddings:
|
||||
return []
|
||||
|
||||
array = np.asarray(float_embeddings, dtype=np.float64)
|
||||
if array.ndim != 2:
|
||||
raise ValueError(
|
||||
f"Expected a 2D batch of embeddings, but got {array.ndim}D input."
|
||||
)
|
||||
|
||||
dim = array.shape[1]
|
||||
if dim % 8 != 0:
|
||||
raise ValueError(
|
||||
"Embedding dimension must be a multiple of 8 for binary "
|
||||
f"embedding types, but got {dim}."
|
||||
)
|
||||
|
||||
packed = np.packbits(array >= 0, axis=-1)
|
||||
if signed:
|
||||
packed = packed.astype(np.int16) - _UNSIGNED_TO_SIGNED_DIFF
|
||||
return packed.tolist()
|
||||
|
||||
|
||||
def _encode_base64_embeddings(
|
||||
|
||||
Reference in New Issue
Block a user