[Frontend] Add routed-experts prompt offset (#51906)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f3c1638927
commit
7553aac77b
@@ -2,14 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, AsyncIterator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
@@ -48,6 +45,7 @@ from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.async_utils import merge_async_iterators
|
||||
from vllm.utils.collection_utils import as_list
|
||||
from vllm.utils.serial_utils import numpy2base64
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -570,17 +568,11 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
# Encode routed_experts for transport. JSON can't carry raw
|
||||
# bytes, so we write the ndarray as a ``.npy`` byte stream
|
||||
# and base64-encode it. ``pybase64`` is ~3x faster than the
|
||||
# stdlib ``base64`` on large payloads thanks to SIMD.
|
||||
routed_experts_b64 = None
|
||||
if output.routed_experts is not None:
|
||||
buf = io.BytesIO()
|
||||
np.save(buf, output.routed_experts)
|
||||
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
|
||||
"ascii"
|
||||
)
|
||||
routed_experts_b64 = (
|
||||
numpy2base64(output.routed_experts)
|
||||
if output.routed_experts is not None
|
||||
else None
|
||||
)
|
||||
|
||||
choice_data = CompletionResponseChoice(
|
||||
index=len(choices),
|
||||
|
||||
Reference in New Issue
Block a user