[Model][Spec Decode] Tap the pre-norm AttnRes mixture as the Kimi K3 DFlash aux state (#50487)
Signed-off-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com> Co-authored-by: Janelle Cai <janelle.cai@modal.com>
This commit is contained in:
co-authored by
Janelle Cai
parent
cdc4824a21
commit
03a8d0b1ed
@@ -207,6 +207,7 @@ if TYPE_CHECKING:
|
||||
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
|
||||
VLLM_MOE_SKIP_PADDING: bool = True
|
||||
VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: bool = False
|
||||
VLLM_KIMI_K3_AUX_ATTN_RES_STREAM: bool = False
|
||||
VLLM_KIMI_K3_GEMM_RS: bool = False
|
||||
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
|
||||
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
|
||||
@@ -1605,6 +1606,13 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT": lambda: bool(
|
||||
int(os.getenv("VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT", "0"))
|
||||
),
|
||||
# Kimi K3 only, and unrelated to the MoE flags above. Tap the pre-norm
|
||||
# AttnRes mixture, rather than the post-mixture sum, as the auxiliary
|
||||
# hidden state handed to a DFlash drafter. This changes the numerics the
|
||||
# speculator sees, so it is off by default while the effect is measured.
|
||||
"VLLM_KIMI_K3_AUX_ATTN_RES_STREAM": lambda: bool(
|
||||
int(os.getenv("VLLM_KIMI_K3_AUX_ATTN_RES_STREAM", "0"))
|
||||
),
|
||||
# Use the SM100 BF16 GEMM-RS kernel for eligible Kimi-K3 sequence-parallel
|
||||
# row-parallel projections. All TP ranks must belong to one NVLink domain.
|
||||
"VLLM_KIMI_K3_GEMM_RS": lambda: bool(int(os.getenv("VLLM_KIMI_K3_GEMM_RS", "0"))),
|
||||
|
||||
Reference in New Issue
Block a user