[AuxOutput] Add block-keyed storage for routed-expert outputs (#45635)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
aoshen02
OpenAI Codex
Roger Wang
Nick Hill
Codex
parent
0aee727ff6
commit
0b7f11a1ee
@@ -35,6 +35,7 @@ from typing_extensions import TypeIs
|
||||
import vllm.envs as envs
|
||||
from vllm.config import (
|
||||
AttentionConfig,
|
||||
AuxOutputConfig,
|
||||
CacheConfig,
|
||||
CompilationConfig,
|
||||
ConfigType,
|
||||
@@ -447,7 +448,9 @@ class EngineArgs:
|
||||
"""Arguments for vLLM engine."""
|
||||
|
||||
model: str = ModelConfig.model
|
||||
enable_return_routed_experts: bool = ModelConfig.enable_return_routed_experts
|
||||
# Public compatibility argument. Canonical runtime state lives in
|
||||
# AuxOutputConfig.
|
||||
enable_return_routed_experts: bool = False
|
||||
return_sampling_mask: bool = ModelConfig.return_sampling_mask
|
||||
model_weights: str = ModelConfig.model_weights
|
||||
served_model_name: str | list[str] | None = ModelConfig.served_model_name
|
||||
@@ -674,6 +677,7 @@ class EngineArgs:
|
||||
structured_outputs_config: StructuredOutputsConfig = get_field(
|
||||
VllmConfig, "structured_outputs_config"
|
||||
)
|
||||
aux_output_config: AuxOutputConfig = get_field(VllmConfig, "aux_output_config")
|
||||
reasoning_parser: str = StructuredOutputsConfig.reasoning_parser
|
||||
reasoning_parser_plugin: str | None = None
|
||||
|
||||
@@ -816,6 +820,10 @@ class EngineArgs:
|
||||
self.compilation_config = CompilationConfig(**self.compilation_config)
|
||||
if isinstance(self.attention_config, dict):
|
||||
self.attention_config = AttentionConfig(**self.attention_config)
|
||||
if isinstance(self.aux_output_config, dict):
|
||||
self.aux_output_config = AuxOutputConfig(**self.aux_output_config)
|
||||
if self.enable_return_routed_experts:
|
||||
self.aux_output_config.enable_return_routed_experts = True
|
||||
if isinstance(self.engram_config, dict):
|
||||
self.engram_config = EngramConfig(**self.engram_config)
|
||||
if isinstance(self.mamba_config, dict):
|
||||
@@ -884,6 +892,7 @@ class EngineArgs:
|
||||
"""Shared CLI arguments for vLLM engine."""
|
||||
# Model arguments
|
||||
model_kwargs = get_kwargs(ModelConfig)
|
||||
aux_output_kwargs = get_kwargs(AuxOutputConfig)
|
||||
model_group = parser.add_argument_group(
|
||||
title="ModelConfig",
|
||||
description=ModelConfig.__doc__,
|
||||
@@ -923,7 +932,7 @@ class EngineArgs:
|
||||
model_group.add_argument("--enforce-eager", **model_kwargs["enforce_eager"])
|
||||
model_group.add_argument(
|
||||
"--enable-return-routed-experts",
|
||||
**model_kwargs["enable_return_routed_experts"],
|
||||
**aux_output_kwargs["enable_return_routed_experts"],
|
||||
)
|
||||
model_group.add_argument(
|
||||
"--return-sampling-mask",
|
||||
@@ -1749,6 +1758,9 @@ class EngineArgs:
|
||||
vllm_group.add_argument(
|
||||
"--structured-outputs-config", **vllm_kwargs["structured_outputs_config"]
|
||||
)
|
||||
vllm_group.add_argument(
|
||||
"--aux-output-config", **vllm_kwargs["aux_output_config"]
|
||||
)
|
||||
vllm_group.add_argument("--profiler-config", **vllm_kwargs["profiler_config"])
|
||||
vllm_group.add_argument(
|
||||
"--optimization-level", **vllm_kwargs["optimization_level"]
|
||||
@@ -1856,7 +1868,6 @@ class EngineArgs:
|
||||
quantization_config=self.quantization_config,
|
||||
allow_deprecated_quantization=self.allow_deprecated_quantization,
|
||||
enforce_eager=self.enforce_eager,
|
||||
enable_return_routed_experts=self.enable_return_routed_experts,
|
||||
return_sampling_mask=self.return_sampling_mask,
|
||||
max_logprobs=self.max_logprobs,
|
||||
logprobs_mode=self.logprobs_mode,
|
||||
@@ -2669,6 +2680,7 @@ class EngineArgs:
|
||||
load_config=load_config,
|
||||
offload_config=offload_config,
|
||||
attention_config=attention_config,
|
||||
aux_output_config=self.aux_output_config,
|
||||
engram_config=self.engram_config,
|
||||
mamba_config=mamba_config,
|
||||
kernel_config=kernel_config,
|
||||
|
||||
Reference in New Issue
Block a user