[vLLM IR] 1/N Implement IR skeleton and rms_norm op (#33825)
Signed-off-by: Luka Govedič <lgovedic@redhat.com> Signed-off-by: Xinyu Chen <xinyu1.chen@intel.com> Signed-off-by: chzhang <chaojun.zhang@intel.com> Signed-off-by: Luka Govedic <luka.govedic@gmail.com> Co-authored-by: Xinyu Chen <xinyu1.chen@intel.com> Co-authored-by: Chaojun Zhang <chaojun.zhang@intel.com> Co-authored-by: Luka Govedič <ProExpertProg@h100-01.nemg-001.lab.rdu2.dc.redhat.com>
This commit is contained in:
co-authored by
Xinyu Chen
Chaojun Zhang
Luka Govedič
parent
0fab52f0aa
commit
40bb175027
@@ -8,7 +8,7 @@ import functools
|
||||
import json
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from dataclasses import MISSING, dataclass, fields, is_dataclass
|
||||
from dataclasses import MISSING, asdict, dataclass, fields, is_dataclass
|
||||
from itertools import permutations
|
||||
from types import UnionType
|
||||
from typing import (
|
||||
@@ -70,7 +70,7 @@ from vllm.config.cache import (
|
||||
PrefixCachingHashAlgo,
|
||||
)
|
||||
from vllm.config.device import Device
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.config.kernel import IrOpPriorityConfig, MoEBackend
|
||||
from vllm.config.lora import MaxLoRARanks
|
||||
from vllm.config.model import (
|
||||
ConvertOption,
|
||||
@@ -401,6 +401,7 @@ class EngineArgs:
|
||||
max_cudagraph_capture_size: int | None = get_field(
|
||||
CompilationConfig, "max_cudagraph_capture_size"
|
||||
)
|
||||
ir_op_priority: IrOpPriorityConfig = get_field(KernelConfig, "ir_op_priority")
|
||||
# Note: Specifying a custom executor backend by passing a class
|
||||
# is intended for expert use only. The API may change without
|
||||
# notice.
|
||||
@@ -657,6 +658,9 @@ class EngineArgs:
|
||||
self.weight_transfer_config = WeightTransferConfig(
|
||||
**self.weight_transfer_config
|
||||
)
|
||||
if isinstance(self.ir_op_priority, dict):
|
||||
self.ir_op_priority = IrOpPriorityConfig(**self.ir_op_priority)
|
||||
|
||||
# Setup plugins
|
||||
from vllm.plugins import load_general_plugins
|
||||
|
||||
@@ -1293,6 +1297,7 @@ class EngineArgs:
|
||||
title="KernelConfig",
|
||||
description=KernelConfig.__doc__,
|
||||
)
|
||||
kernel_group.add_argument("--ir-op-priority", **kernel_kwargs["ir_op_priority"])
|
||||
kernel_group.add_argument(
|
||||
"--enable-flashinfer-autotune",
|
||||
**kernel_kwargs["enable_flashinfer_autotune"],
|
||||
@@ -1917,6 +1922,22 @@ class EngineArgs:
|
||||
if self.moe_backend != "auto":
|
||||
kernel_config.moe_backend = self.moe_backend
|
||||
|
||||
# Transfer top-level ir_op_priority into KernelConfig.ir_op_priority
|
||||
for op_name, op_priority in asdict(self.ir_op_priority).items():
|
||||
# Empty means unset
|
||||
if not op_priority:
|
||||
continue
|
||||
|
||||
# Priority cannot be set 2x for the same op
|
||||
if getattr(kernel_config.ir_op_priority, op_name):
|
||||
raise ValueError(
|
||||
f"Op priority for {op_name} specified via both ir_op_priority "
|
||||
f"and KernelConfig.ir_op_priority, only one allowed at a time."
|
||||
)
|
||||
|
||||
# Set the attribute
|
||||
setattr(kernel_config.ir_op_priority, op_name, op_priority)
|
||||
|
||||
load_config = self.create_load_config()
|
||||
|
||||
# Pass reasoning_parser into StructuredOutputsConfig
|
||||
|
||||
Reference in New Issue
Block a user