[Frontend] Add routed-experts prompt offset (#51906)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
aoshen02
2026-08-13 08:09:00 -07:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent f3c1638927
commit 7553aac77b
13 changed files with 114 additions and 61 deletions
+6 -14
View File
@@ -2,14 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import io
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from typing import cast
import numpy as np
import pybase64 as base64
from fastapi import Request
from vllm.engine.protocol import EngineClient
@@ -48,6 +45,7 @@ from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.utils.async_utils import merge_async_iterators
from vllm.utils.collection_utils import as_list
from vllm.utils.serial_utils import numpy2base64
logger = init_logger(__name__)
@@ -570,17 +568,11 @@ class OpenAIServingCompletion(GenerateBaseServing):
else:
logprobs = None
# Encode routed_experts for transport. JSON can't carry raw
# bytes, so we write the ndarray as a ``.npy`` byte stream
# and base64-encode it. ``pybase64`` is ~3x faster than the
# stdlib ``base64`` on large payloads thanks to SIMD.
routed_experts_b64 = None
if output.routed_experts is not None:
buf = io.BytesIO()
np.save(buf, output.routed_experts)
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
"ascii"
)
routed_experts_b64 = (
numpy2base64(output.routed_experts)
if output.routed_experts is not None
else None
)
choice_data = CompletionResponseChoice(
index=len(choices),