[AuxOutput] Add block-keyed storage for routed-expert outputs (#45635)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
aoshen02
OpenAI Codex
Roger Wang
Nick Hill
Codex
parent
0aee727ff6
commit
0b7f11a1ee
@@ -12,15 +12,13 @@ from ...utils import RemoteOpenAIServer
|
||||
MODEL_NAME = "TitanML/tiny-mixtral"
|
||||
|
||||
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
|
||||
# The published config has sliding_window=4096, which produces
|
||||
# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a
|
||||
# FullAttentionSpec group, so we override sliding_window=null below.
|
||||
# Its published sliding window is incompatible with AuxOutput retention.
|
||||
NUM_LOCAL_EXPERTS = 8
|
||||
NUM_EXPERTS_PER_TOK = 2
|
||||
NUM_HIDDEN_LAYERS = 2
|
||||
|
||||
|
||||
def assert_valid_routed_experts(encoded: str | None) -> None:
|
||||
def decode_routed_experts(encoded: str | None) -> np.ndarray:
|
||||
assert encoded is not None
|
||||
routed_experts = np.load(io.BytesIO(base64.b64decode(encoded)))
|
||||
assert routed_experts.ndim == 3
|
||||
@@ -30,6 +28,7 @@ def assert_valid_routed_experts(encoded: str | None) -> None:
|
||||
assert topk == NUM_EXPERTS_PER_TOK
|
||||
assert (routed_experts >= 0).all()
|
||||
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
|
||||
return routed_experts
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
@@ -44,7 +43,11 @@ def server():
|
||||
"--hf-overrides",
|
||||
'{"sliding_window": null}',
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_USE_V2_MODEL_RUNNER": "1"},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@@ -63,4 +66,8 @@ async def test_routed_experts(server):
|
||||
choice = result.model_dump()["choices"][0]
|
||||
|
||||
assert choice["token_ids"] is not None
|
||||
assert_valid_routed_experts(choice["routed_experts"])
|
||||
assert result.usage is not None
|
||||
routed_experts = decode_routed_experts(choice["routed_experts"])
|
||||
assert len(routed_experts) == (
|
||||
result.usage.prompt_tokens + len(choice["token_ids"]) - 1
|
||||
)
|
||||
|
||||
@@ -15,9 +15,7 @@ MODEL_NAME = "TitanML/tiny-mixtral"
|
||||
GEN_ENDPOINT = "/inference/v1/generate"
|
||||
|
||||
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
|
||||
# The published config has sliding_window=4096, which produces
|
||||
# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a
|
||||
# FullAttentionSpec group, so we override sliding_window=null below.
|
||||
# Its published sliding window is incompatible with AuxOutput retention.
|
||||
NUM_LOCAL_EXPERTS = 8
|
||||
NUM_EXPERTS_PER_TOK = 2
|
||||
NUM_HIDDEN_LAYERS = 2
|
||||
@@ -36,7 +34,13 @@ def server():
|
||||
'{"sliding_window": null}',
|
||||
"--enable-scale-out",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={
|
||||
"VLLM_USE_V2_MODEL_RUNNER": "1",
|
||||
},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@@ -56,9 +60,10 @@ async def client(server: RemoteOpenAIServer):
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_routed_experts(client):
|
||||
"""Test that /inference/v1/generate returns routed_experts when enabled."""
|
||||
prompt_token_ids = [1, 2, 3]
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"token_ids": prompt_token_ids,
|
||||
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
|
||||
"stream": False,
|
||||
}
|
||||
@@ -80,3 +85,4 @@ async def test_generate_routed_experts(client):
|
||||
assert topk == NUM_EXPERTS_PER_TOK
|
||||
assert (routed_experts >= 0).all()
|
||||
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
|
||||
assert num_tokens == len(prompt_token_ids) + len(choice["token_ids"]) - 1
|
||||
|
||||
Reference in New Issue
Block a user