[Frontend] Add routed-experts prompt offset (#51906)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f3c1638927
commit
7553aac77b
@@ -114,7 +114,7 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
|
||||
token_ids: list[int] | None = None
|
||||
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
|
||||
# (numpy serialization). Shape after decode:
|
||||
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16
|
||||
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16/int32
|
||||
# ``num_tokens - 1`` because the last sampled token has not been
|
||||
# forwarded yet and therefore has no routing data.
|
||||
# Decode:
|
||||
@@ -422,6 +422,11 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
"need to map generated text back to input tokens."
|
||||
),
|
||||
)
|
||||
routed_experts_prompt_start: int = Field(
|
||||
default=0,
|
||||
ge=0,
|
||||
description="Skip the first N prompt tokens from returned routed-expert data.",
|
||||
)
|
||||
return_token_offsets: bool | None = Field(
|
||||
default=False,
|
||||
description=(
|
||||
@@ -741,6 +746,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
extra_args=extra_args or None,
|
||||
skip_clone=True, # Created fresh per request, safe to skip clone
|
||||
repetition_detection=self.repetition_detection,
|
||||
routed_experts_prompt_start=self.routed_experts_prompt_start,
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
|
||||
@@ -2,15 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, AsyncIterator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
from http import HTTPStatus
|
||||
from typing import Any, Final, cast
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
@@ -64,6 +61,7 @@ from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.collection_utils import as_list
|
||||
from vllm.utils.serial_utils import numpy2base64
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -1007,15 +1005,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
and output.finish_reason == "stop"
|
||||
)
|
||||
|
||||
# Encode routed_experts for transport. JSON can't carry raw
|
||||
# bytes, so we write the ndarray as a ``.npy`` byte stream
|
||||
# and base64-encode it. ``pybase64`` is ~3x faster than the
|
||||
# stdlib ``base64`` on large payloads thanks to SIMD.
|
||||
routed_experts_b64 = None
|
||||
if output.routed_experts is not None:
|
||||
buf = io.BytesIO()
|
||||
np.save(buf, output.routed_experts)
|
||||
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode("ascii")
|
||||
routed_experts_b64 = (
|
||||
numpy2base64(output.routed_experts)
|
||||
if output.routed_experts is not None
|
||||
else None
|
||||
)
|
||||
|
||||
choice_data = ChatCompletionResponseChoice(
|
||||
index=output.index,
|
||||
|
||||
@@ -175,6 +175,11 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
"need to map generated text back to input tokens."
|
||||
),
|
||||
)
|
||||
routed_experts_prompt_start: int = Field(
|
||||
default=0,
|
||||
ge=0,
|
||||
description="Skip the first N prompt tokens from returned routed-expert data.",
|
||||
)
|
||||
return_token_offsets: bool | None = Field(
|
||||
default=False,
|
||||
description=(
|
||||
@@ -394,6 +399,7 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
skip_clone=True, # Created fresh per request, safe to skip clone
|
||||
repetition_detection=self.repetition_detection,
|
||||
thinking_token_budget=self.thinking_token_budget,
|
||||
routed_experts_prompt_start=self.routed_experts_prompt_start,
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@@ -611,7 +617,7 @@ class CompletionResponseChoice(OpenAIBaseModel):
|
||||
prompt_token_ids: list[int] | None = None # For prompt
|
||||
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
|
||||
# (numpy serialization). Shape after decode:
|
||||
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16
|
||||
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16/int32
|
||||
# ``num_tokens - 1`` because the last sampled token has not been
|
||||
# forwarded yet and therefore has no routing data.
|
||||
# Decode:
|
||||
|
||||
@@ -2,14 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, AsyncIterator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
@@ -48,6 +45,7 @@ from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.async_utils import merge_async_iterators
|
||||
from vllm.utils.collection_utils import as_list
|
||||
from vllm.utils.serial_utils import numpy2base64
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -570,17 +568,11 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
# Encode routed_experts for transport. JSON can't carry raw
|
||||
# bytes, so we write the ndarray as a ``.npy`` byte stream
|
||||
# and base64-encode it. ``pybase64`` is ~3x faster than the
|
||||
# stdlib ``base64`` on large payloads thanks to SIMD.
|
||||
routed_experts_b64 = None
|
||||
if output.routed_experts is not None:
|
||||
buf = io.BytesIO()
|
||||
np.save(buf, output.routed_experts)
|
||||
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
|
||||
"ascii"
|
||||
)
|
||||
routed_experts_b64 = (
|
||||
numpy2base64(output.routed_experts)
|
||||
if output.routed_experts is not None
|
||||
else None
|
||||
)
|
||||
|
||||
choice_data = CompletionResponseChoice(
|
||||
index=len(choices),
|
||||
|
||||
@@ -202,7 +202,7 @@ class GenerateResponseChoice(BaseModel):
|
||||
token_ids: list[int] | None = None
|
||||
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
|
||||
# (numpy serialization). Shape after decode:
|
||||
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16
|
||||
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16/int32
|
||||
# ``num_tokens - 1`` because the last sampled token has not been
|
||||
# forwarded yet and therefore has no routing data.
|
||||
# Decode:
|
||||
|
||||
@@ -3,14 +3,11 @@
|
||||
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import time
|
||||
from collections.abc import AsyncGenerator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
@@ -46,6 +43,7 @@ from vllm.outputs import RequestOutput
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.utils.collection_utils import as_list
|
||||
from vllm.utils.serial_utils import numpy2base64
|
||||
|
||||
from .mm_serde import decode_mm_kwargs_item
|
||||
from .protocol import (
|
||||
@@ -303,17 +301,11 @@ class ServingTokens(GenerateBaseServing):
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
# Encode routed_experts for transport. JSON can't carry raw
|
||||
# bytes, so we write the ndarray as a ``.npy`` byte stream
|
||||
# and base64-encode it. ``pybase64`` is ~3x faster than the
|
||||
# stdlib ``base64`` on large payloads thanks to SIMD.
|
||||
# This is the only base64 hop in the pipeline -- the
|
||||
# engine<->API-server link is binary msgpack + zmq.
|
||||
routed_experts_b64 = None
|
||||
if output.routed_experts is not None:
|
||||
buf = io.BytesIO()
|
||||
np.save(buf, output.routed_experts)
|
||||
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode("ascii")
|
||||
routed_experts_b64 = (
|
||||
numpy2base64(output.routed_experts)
|
||||
if output.routed_experts is not None
|
||||
else None
|
||||
)
|
||||
|
||||
sampling_mask = None
|
||||
if output.sampling_mask is not None:
|
||||
@@ -435,13 +427,11 @@ class ServingTokens(GenerateBaseServing):
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
routed_experts_b64 = None
|
||||
if output.routed_experts is not None:
|
||||
buf = io.BytesIO()
|
||||
np.save(buf, output.routed_experts)
|
||||
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
|
||||
"ascii"
|
||||
)
|
||||
routed_experts_b64 = (
|
||||
numpy2base64(output.routed_experts)
|
||||
if output.routed_experts is not None
|
||||
else None
|
||||
)
|
||||
|
||||
chunk = GenerateStreamResponse(
|
||||
request_id=request_id,
|
||||
|
||||
Reference in New Issue
Block a user