[RFC] Replace shared-memory routed experts with ModelRunnerOutput transfer and HTTP support (#39568)
Signed-off-by: xhx1022 <1737006628@qq.com> Signed-off-by: arlenxu <arlenxu@tencent.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: arlenxu <arlenxu@tencent.com> Co-authored-by: Junjie Zhang <junj.jay.zhang@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
arlenxu
Junjie Zhang
parent
2317682f95
commit
c7560af424
@@ -160,6 +160,16 @@ class GenerateResponseChoice(BaseModel):
|
||||
# per OpenAI spec this is the default
|
||||
finish_reason: str | None = "stop"
|
||||
token_ids: list[int] | None = None
|
||||
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
|
||||
# (numpy serialization). Shape after decode:
|
||||
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16
|
||||
# ``num_tokens - 1`` because the last sampled token has not been
|
||||
# forwarded yet and therefore has no routing data.
|
||||
# Decode:
|
||||
# np.load(io.BytesIO(base64.b64decode(s)))
|
||||
# ``None`` if (a) the request was aborted before any forward pass,
|
||||
# or (b) ``enable_return_routed_experts`` is off server-side.
|
||||
routed_experts: str | None = None
|
||||
|
||||
|
||||
class GenerateResponseStreamChoice(BaseModel):
|
||||
|
||||
@@ -3,10 +3,13 @@
|
||||
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import time
|
||||
from collections.abc import AsyncGenerator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
@@ -255,11 +258,24 @@ class ServingTokens(OpenAIServing):
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
# Encode routed_experts for transport. JSON can't carry raw
|
||||
# bytes, so we write the ndarray as a ``.npy`` byte stream
|
||||
# and base64-encode it. ``pybase64`` is ~3x faster than the
|
||||
# stdlib ``base64`` on large payloads thanks to SIMD.
|
||||
# This is the only base64 hop in the pipeline -- the
|
||||
# engine<->API-server link is binary msgpack + zmq.
|
||||
routed_experts_b64 = None
|
||||
if output.routed_experts is not None:
|
||||
buf = io.BytesIO()
|
||||
np.save(buf, output.routed_experts)
|
||||
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode("ascii")
|
||||
|
||||
choice_data = GenerateResponseChoice(
|
||||
index=output.index,
|
||||
logprobs=logprobs,
|
||||
finish_reason=output.finish_reason if output.finish_reason else "stop",
|
||||
token_ids=as_list(output.token_ids),
|
||||
routed_experts=routed_experts_b64,
|
||||
)
|
||||
|
||||
choices.append(choice_data)
|
||||
|
||||
Reference in New Issue
Block a user