From 103c419f048474d757c752524abb1e155dd01b9a Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Fri, 14 Aug 2026 02:15:59 -0700 Subject: [PATCH] [Perf][Frontend] Vectorize Cohere binary embedding bit-packing (#52277) Signed-off-by: Fangchen Li Co-authored-by: Claude Opus 5 (1M context) --- .../pooling/embed/test_protocol.py | 35 ++++++++++++++ vllm/entrypoints/pooling/embed/protocol.py | 46 +++++++++---------- 2 files changed, 57 insertions(+), 24 deletions(-) diff --git a/tests/entrypoints/pooling/embed/test_protocol.py b/tests/entrypoints/pooling/embed/test_protocol.py index 9d3416b772..91c853f910 100644 --- a/tests/entrypoints/pooling/embed/test_protocol.py +++ b/tests/entrypoints/pooling/embed/test_protocol.py @@ -75,6 +75,41 @@ class TestBuildTypedEmbeddingsBinary: # 0.0 >= 0 is True, so bit=1 for all => 127 (signed) assert result.binary[0] == [127] + def test_negative_zero_treated_as_positive(self): + """Packing follows ``value >= 0``, not the IEEE-754 sign bit. + + ``-0.0 >= 0`` is True, so these bits must be set. An implementation + reading the sign bit instead would pack -128 here. + """ + result = build_typed_embeddings([[-0.0] * 8], ["binary"]) + assert result.binary is not None + assert result.binary[0] == [127] + + def test_negative_denormal_treated_as_negative(self): + """Sign is decided at float64 precision. + + The smallest float64 denormal underflows to -0.0 in float32, where + ``>= 0`` is True; narrowing the comparison would wrongly pack 127. + """ + result = build_typed_embeddings([[-5e-324] * 8], ["binary"]) + assert result.binary is not None + assert result.binary[0] == [-128] + + def test_empty_input(self): + result = build_typed_embeddings([], ["binary", "ubinary"]) + assert result.binary == [] + assert result.ubinary == [] + + @pytest.mark.parametrize("embs", [[0.1] * 8, [[[0.1] * 8] * 2]], ids=["1d", "3d"]) + def test_non_2d_input_raises(self, embs): + """A malformed batch must raise rather than pack along the last axis. + + Packing a 1D or 3D input would silently emit wrongly shaped + ``binary``/``ubinary`` values instead of reporting an error. + """ + with pytest.raises(ValueError, match="2D batch"): + build_typed_embeddings(embs, ["binary"]) + def test_non_multiple_of_8_raises(self): embs = [[0.1] * 7] with pytest.raises(ValueError, match="multiple of 8"): diff --git a/vllm/entrypoints/pooling/embed/protocol.py b/vllm/entrypoints/pooling/embed/protocol.py index 96a87fea20..d319131a1d 100644 --- a/vllm/entrypoints/pooling/embed/protocol.py +++ b/vllm/entrypoints/pooling/embed/protocol.py @@ -12,6 +12,7 @@ import time from collections.abc import Sequence from typing import Annotated, Any, Literal, TypeAlias +import numpy as np import pybase64 as base64 from pydantic import BaseModel, BeforeValidator, Field, model_validator @@ -298,31 +299,28 @@ def _pack_binary_embeddings( ) -> list[list[int]]: """Bit-pack float embeddings: positive -> 1, negative -> 0. - Each bit is shifted left by ``7 - idx%8``, and every 8 bits are packed - into one byte. + Bits are packed MSB-first, eight per byte. """ - result: list[list[int]] = [] - for embedding in float_embeddings: - dim = len(embedding) - if dim % 8 != 0: - raise ValueError( - "Embedding dimension must be a multiple of 8 for binary " - f"embedding types, but got {dim}." - ) - packed_len = dim // 8 - packed: list[int] = [] - byte_val = 0 - for idx, value in enumerate(embedding): - bit = 1 if value >= 0 else 0 - byte_val += bit << (7 - idx % 8) - if (idx + 1) % 8 == 0: - if signed: - byte_val -= _UNSIGNED_TO_SIGNED_DIFF - packed.append(byte_val) - byte_val = 0 - assert len(packed) == packed_len - result.append(packed) - return result + if not float_embeddings: + return [] + + array = np.asarray(float_embeddings, dtype=np.float64) + if array.ndim != 2: + raise ValueError( + f"Expected a 2D batch of embeddings, but got {array.ndim}D input." + ) + + dim = array.shape[1] + if dim % 8 != 0: + raise ValueError( + "Embedding dimension must be a multiple of 8 for binary " + f"embedding types, but got {dim}." + ) + + packed = np.packbits(array >= 0, axis=-1) + if signed: + packed = packed.astype(np.int16) - _UNSIGNED_TO_SIGNED_DIFF + return packed.tolist() def _encode_base64_embeddings(