Update BGE-M3 token expectations for leading spaces (#49269)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
aoshen02
Codex
parent
6bcda970fd
commit
d9aa35161d
@@ -48,12 +48,12 @@ def _check_dense_embedding(data, index=0):
|
||||
def _check_sparse_embedding(data, check_tokens=False):
|
||||
expected_weights = [
|
||||
{"token_id": 32, "weight": 0.0552978515625, "token": "?"},
|
||||
{"token_id": 70, "weight": 0.09808349609375, "token": "the"},
|
||||
{"token_id": 83, "weight": 0.08154296875, "token": "is"},
|
||||
{"token_id": 111, "weight": 0.11810302734375, "token": "of"},
|
||||
{"token_id": 4865, "weight": 0.1171875, "token": "What"},
|
||||
{"token_id": 9942, "weight": 0.292236328125, "token": "France"},
|
||||
{"token_id": 10323, "weight": 0.2802734375, "token": "capital"},
|
||||
{"token_id": 70, "weight": 0.09808349609375, "token": " the"},
|
||||
{"token_id": 83, "weight": 0.08154296875, "token": " is"},
|
||||
{"token_id": 111, "weight": 0.11810302734375, "token": " of"},
|
||||
{"token_id": 4865, "weight": 0.1171875, "token": " What"},
|
||||
{"token_id": 9942, "weight": 0.292236328125, "token": " France"},
|
||||
{"token_id": 10323, "weight": 0.2802734375, "token": " capital"},
|
||||
]
|
||||
expected_embed = {x["token_id"]: x for x in expected_weights}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user