[kernel] Integrate FlashInfer BF16 CuTeDSL Low Latency GEMM (#50572)
Signed-off-by: jiahanc <173873397+jiahanc@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Jee Jee Li
parent
f79a2f5582
commit
7c877062ac
@@ -10,7 +10,7 @@ from torch.nn.parameter import Parameter
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config import get_current_vllm_config, get_current_vllm_config_or_none
|
||||
from vllm.distributed import (
|
||||
divide,
|
||||
get_tensor_model_parallel_rank,
|
||||
@@ -166,6 +166,13 @@ class LinearMethodBase(QuantizeMethodBase):
|
||||
class UnquantizedLinearMethod(LinearMethodBase):
|
||||
"""Linear method without quantization."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
config = get_current_vllm_config_or_none()
|
||||
linear_backend = (
|
||||
config.kernel_config.linear_backend if config is not None else "auto"
|
||||
)
|
||||
self._gemm_impl = dispatch_unquantized_gemm(linear_backend)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -215,7 +222,7 @@ class UnquantizedLinearMethod(LinearMethodBase):
|
||||
) -> torch.Tensor:
|
||||
if envs.VLLM_BATCH_INVARIANT and current_platform.is_cuda_alike():
|
||||
return linear_batch_invariant(x, layer.weight, bias)
|
||||
return dispatch_unquantized_gemm()(layer, x, layer.weight, bias)
|
||||
return self._gemm_impl(layer, x, layer.weight, bias)
|
||||
|
||||
|
||||
class LinearBase(PluggableLayer):
|
||||
|
||||
Reference in New Issue
Block a user