[compile] Allow strings in custom ops without regressing compilation times (#38123)
Signed-off-by: Richard Zou <zou3519@gmail.com>
This commit is contained in:
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import _USE_LAYERNAME, _encode_layer_name
|
||||
|
||||
from ..vllm_inductor_pass import VllmFusionPatternMatcherPass, VllmPatternReplacement
|
||||
from .matcher_utils import MatcherQuantFP8
|
||||
@@ -53,21 +54,43 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
|
||||
@property
|
||||
def pattern(self) -> Callable[..., torch.Tensor]:
|
||||
def _pattern(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
output_attn: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
kv_cache_dummy_dep: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# When _USE_LAYERNAME is enabled (torch >= 2.11), layer_name is
|
||||
# passed as an explicit pattern input so the pattern matcher
|
||||
# treats it as a wildcard matching hoisted LayerName placeholders.
|
||||
# Otherwise it stays as a closure constant (original behavior).
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _pattern_with_ln( # type: ignore[misc]
|
||||
q, k, v, output_attn, scale, kv_cache_dummy_dep, layer_name
|
||||
):
|
||||
at1 = auto_functionalized(
|
||||
ATTN_OP,
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
attn_out_view = RESHAPE_OP(
|
||||
at1[1], [q.shape[0], self._num_heads * self._head_size]
|
||||
)
|
||||
return self._quant_matcher(attn_out_view, scale)[0]
|
||||
|
||||
return _pattern_with_ln
|
||||
|
||||
def _pattern(q, k, v, output_attn, scale, kv_cache_dummy_dep):
|
||||
at1 = auto_functionalized(
|
||||
ATTN_OP,
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
output=output_attn,
|
||||
layer_name=self._layer_name,
|
||||
layer_name=_ln,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
@@ -81,14 +104,34 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
|
||||
@property
|
||||
def replacement(self) -> Callable[..., torch.Tensor]:
|
||||
def _replacement(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
output_attn: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
kv_cache_dummy_dep: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _replacement_with_ln( # type: ignore[misc]
|
||||
q, k, v, output_attn, scale, kv_cache_dummy_dep, layer_name
|
||||
):
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._num_heads, self._head_size],
|
||||
dtype=FP8_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
at1 = auto_functionalized(
|
||||
ATTN_OP,
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=scale,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size])
|
||||
|
||||
return _replacement_with_ln
|
||||
|
||||
def _replacement(q, k, v, output_attn, scale, kv_cache_dummy_dep):
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._num_heads, self._head_size],
|
||||
dtype=FP8_DTYPE,
|
||||
@@ -100,7 +143,7 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
key=k,
|
||||
value=v,
|
||||
output=output_attn,
|
||||
layer_name=self._layer_name,
|
||||
layer_name=_ln,
|
||||
output_scale=scale,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
@@ -113,7 +156,7 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
dtype = self._dtype
|
||||
num_heads = self._num_heads
|
||||
head_size = self._head_size
|
||||
return [
|
||||
inputs: list = [
|
||||
self.empty(5, num_heads, head_size, dtype=dtype), # q
|
||||
self.empty(5, num_heads, head_size, dtype=dtype), # k
|
||||
self.empty(5, num_heads, head_size, dtype=dtype), # v
|
||||
@@ -121,6 +164,9 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
self.empty_fp32(1, 1), # scale
|
||||
self.empty(0, dtype=dtype), # kv_cache_dummy_dep
|
||||
]
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(_encode_layer_name(self._layer_name))
|
||||
return inputs
|
||||
|
||||
|
||||
class AttnNvfp4QuantPattern(
|
||||
@@ -144,23 +190,64 @@ class AttnNvfp4QuantPattern(
|
||||
|
||||
@property
|
||||
def pattern(self) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _pattern_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
output_attn,
|
||||
output_quant,
|
||||
output_scale,
|
||||
input_scale,
|
||||
kv_cache_dummy_dep,
|
||||
layer_name,
|
||||
):
|
||||
at1 = auto_functionalized(
|
||||
ATTN_OP,
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
attn_out_view = RESHAPE_OP(
|
||||
at1[1], [q.shape[0], self._num_heads * self._head_size]
|
||||
)
|
||||
at2 = auto_functionalized(
|
||||
self._QUANT_OP,
|
||||
input=attn_out_view,
|
||||
input_scale=input_scale,
|
||||
is_sf_swizzled_layout=True,
|
||||
output=output_quant,
|
||||
output_scale=output_scale,
|
||||
)
|
||||
return at2[1], torch.ops.aten.view.dtype(at2[2], FP8_DTYPE)
|
||||
|
||||
return _pattern_with_ln
|
||||
|
||||
def _pattern(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
output_attn: torch.Tensor,
|
||||
output_quant: torch.Tensor,
|
||||
output_scale: torch.Tensor,
|
||||
input_scale: torch.Tensor,
|
||||
kv_cache_dummy_dep: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
output_attn,
|
||||
output_quant,
|
||||
output_scale,
|
||||
input_scale,
|
||||
kv_cache_dummy_dep,
|
||||
):
|
||||
at1 = auto_functionalized(
|
||||
ATTN_OP,
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
output=output_attn,
|
||||
layer_name=self._layer_name,
|
||||
layer_name=_ln,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
@@ -176,42 +263,80 @@ class AttnNvfp4QuantPattern(
|
||||
output=output_quant,
|
||||
output_scale=output_scale,
|
||||
)
|
||||
output_scale_view = torch.ops.aten.view.dtype(at2[2], FP8_DTYPE)
|
||||
return at2[1], output_scale_view
|
||||
return at2[1], torch.ops.aten.view.dtype(at2[2], FP8_DTYPE)
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _replacement_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
output_attn,
|
||||
_output_quant,
|
||||
output_scale,
|
||||
input_scale,
|
||||
kv_cache_dummy_dep,
|
||||
layer_name,
|
||||
):
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._num_heads, self._head_size // 2],
|
||||
dtype=FP4_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
osv = torch.ops.aten.view.dtype(output_scale, FP8_DTYPE)
|
||||
at2 = auto_functionalized(
|
||||
ATTN_OP,
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=input_scale,
|
||||
output_block_scale=osv,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return RESHAPE_OP(
|
||||
at2[1], [-1, self._num_heads * self._head_size // 2]
|
||||
), at2[2]
|
||||
|
||||
return _replacement_with_ln
|
||||
|
||||
def _replacement(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
output_attn: torch.Tensor,
|
||||
_output_quant: torch.Tensor,
|
||||
output_scale: torch.Tensor,
|
||||
input_scale: torch.Tensor,
|
||||
kv_cache_dummy_dep: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
output_attn,
|
||||
_output_quant,
|
||||
output_scale,
|
||||
input_scale,
|
||||
kv_cache_dummy_dep,
|
||||
):
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._num_heads, self._head_size // 2],
|
||||
dtype=FP4_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
output_scale_view = torch.ops.aten.view.dtype(output_scale, FP8_DTYPE)
|
||||
osv = torch.ops.aten.view.dtype(output_scale, FP8_DTYPE)
|
||||
at2 = auto_functionalized(
|
||||
ATTN_OP,
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
output=output_attn,
|
||||
layer_name=self._layer_name,
|
||||
layer_name=_ln,
|
||||
output_scale=input_scale,
|
||||
output_block_scale=output_scale_view,
|
||||
output_block_scale=osv,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
output = RESHAPE_OP(at2[1], [-1, self._num_heads * self._head_size // 2])
|
||||
return output, at2[2]
|
||||
return RESHAPE_OP(
|
||||
at2[1], [-1, self._num_heads * self._head_size // 2]
|
||||
), at2[2]
|
||||
|
||||
return _replacement
|
||||
|
||||
@@ -219,18 +344,19 @@ class AttnNvfp4QuantPattern(
|
||||
dtype = self._dtype
|
||||
num_heads = self._num_heads
|
||||
head_size = self._head_size
|
||||
return [
|
||||
inputs: list = [
|
||||
self.empty_bf16(5, num_heads, head_size), # q
|
||||
self.empty_bf16(5, num_heads, head_size), # k
|
||||
self.empty_bf16(5, num_heads, head_size), # v
|
||||
self.empty_bf16(5, num_heads, head_size), # output_attn
|
||||
self.empty(5, num_heads * head_size // 2, dtype=FP4_DTYPE), # output_quant
|
||||
self.empty_i32(
|
||||
128, round_up(num_heads * head_size // 16, 4)
|
||||
), # output_scale
|
||||
self.empty(5, num_heads * head_size // 2, dtype=FP4_DTYPE),
|
||||
self.empty_i32(128, round_up(num_heads * head_size // 16, 4)),
|
||||
self.empty_fp32(1, 1), # input_scale
|
||||
self.empty(0, dtype=dtype), # kv_cache_dummy_dep
|
||||
]
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(_encode_layer_name(self._layer_name))
|
||||
return inputs
|
||||
|
||||
|
||||
class AttnQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
@@ -259,13 +385,19 @@ class AttnQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"so no fusion patterns were registered."
|
||||
)
|
||||
|
||||
# When _USE_LAYERNAME is enabled, layer_name is a wildcard so all
|
||||
# layers produce the same pattern — register once then break.
|
||||
for layer in layers:
|
||||
if layer.impl.fused_output_quant_supported(_FP8_QUANT_KEY):
|
||||
self.register(AttnFp8StaticQuantPattern(layer, dtype))
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"):
|
||||
for layer in layers:
|
||||
if layer.impl.fused_output_quant_supported(kNvfp4Dynamic):
|
||||
self.register(AttnNvfp4QuantPattern(layer, dtype))
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
self.dump_patterns(config, self.pm_pass)
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import _USE_LAYERNAME, _encode_layer_name
|
||||
|
||||
from ..vllm_inductor_pass import VllmFusionPatternMatcherPass, VllmPatternReplacement
|
||||
from .matcher_utils import MatcherQuantFP8
|
||||
@@ -49,21 +50,43 @@ class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
|
||||
@property
|
||||
def pattern(self) -> Callable[..., torch.Tensor]:
|
||||
def _pattern(
|
||||
q: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
output_attn: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
kv_cache_dummy_dep: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _pattern_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
scale,
|
||||
kv_cache_dummy_dep,
|
||||
layer_name,
|
||||
):
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
# MLA output is already 2D (T, N*V), no reshape needed
|
||||
return self._quant_matcher(at1[1], scale)[0]
|
||||
|
||||
return _pattern_with_ln
|
||||
|
||||
def _pattern(q, kv_c_normed, k_pe, output_attn, scale, kv_cache_dummy_dep):
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=self._layer_name,
|
||||
layer_name=_ln,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
@@ -75,14 +98,41 @@ class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
|
||||
@property
|
||||
def replacement(self) -> Callable[..., torch.Tensor]:
|
||||
def _replacement(
|
||||
q: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
output_attn: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
kv_cache_dummy_dep: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _replacement_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
scale,
|
||||
kv_cache_dummy_dep,
|
||||
layer_name,
|
||||
):
|
||||
# MLA output in quant_dtype
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim],
|
||||
dtype=FP8_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=scale,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return at1[1]
|
||||
|
||||
return _replacement_with_ln
|
||||
|
||||
def _replacement(q, kv_c_normed, k_pe, output_attn, scale, kv_cache_dummy_dep):
|
||||
# MLA output in quant_dtype
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim],
|
||||
@@ -95,7 +145,7 @@ class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=self._layer_name,
|
||||
layer_name=_ln,
|
||||
output_scale=scale,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
@@ -105,7 +155,7 @@ class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
return _replacement
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
return [
|
||||
inputs: list = [
|
||||
self.empty(5, self._num_heads, self._qk_head_dim, dtype=self._dtype),
|
||||
self.empty(5, self._kv_lora_rank, dtype=self._dtype),
|
||||
self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype),
|
||||
@@ -113,6 +163,9 @@ class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
self.empty_fp32(1, 1),
|
||||
self.empty(0, dtype=self._dtype),
|
||||
]
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(_encode_layer_name(self._layer_name))
|
||||
return inputs
|
||||
|
||||
|
||||
class MLAAttnNvfp4QuantPattern(
|
||||
@@ -141,21 +194,56 @@ class MLAAttnNvfp4QuantPattern(
|
||||
def pattern(
|
||||
self,
|
||||
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _pattern_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
input_scale,
|
||||
kv_cache_dummy_dep,
|
||||
layer_name,
|
||||
):
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
output_quant, output_scale = create_fp4_output_tensors(
|
||||
at1[1].shape[0], at1[1].shape[1], at1[1].device, True
|
||||
)
|
||||
at2 = auto_functionalized(
|
||||
self._QUANT_OP,
|
||||
input=at1[1],
|
||||
input_scale=input_scale,
|
||||
is_sf_swizzled_layout=True,
|
||||
output=output_quant,
|
||||
output_scale=output_scale,
|
||||
)
|
||||
output_scale_view = torch.ops.aten.view.dtype(at2[2], FP8_DTYPE)
|
||||
return at2[1], output_scale_view
|
||||
|
||||
return _pattern_with_ln
|
||||
|
||||
def _pattern(
|
||||
q: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
output_attn: torch.Tensor,
|
||||
input_scale: torch.Tensor,
|
||||
kv_cache_dummy_dep: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
q, kv_c_normed, k_pe, output_attn, input_scale, kv_cache_dummy_dep
|
||||
):
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=self._layer_name,
|
||||
layer_name=_ln,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
@@ -182,14 +270,47 @@ class MLAAttnNvfp4QuantPattern(
|
||||
def replacement(
|
||||
self,
|
||||
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _replacement_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
input_scale,
|
||||
kv_cache_dummy_dep,
|
||||
layer_name,
|
||||
):
|
||||
# MLA output in quant_dtype (FP4 packed as uint8)
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim // 2],
|
||||
dtype=FP4_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
output_scale = create_fp4_output_tensors(
|
||||
q.shape[0], self._output_dim, q.device, True
|
||||
)[1]
|
||||
output_scale_view = torch.ops.aten.view.dtype(output_scale, FP8_DTYPE)
|
||||
at2 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=input_scale,
|
||||
output_block_scale=output_scale_view,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return at2[1], at2[2]
|
||||
|
||||
return _replacement_with_ln
|
||||
|
||||
def _replacement(
|
||||
q: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
output_attn: torch.Tensor,
|
||||
input_scale: torch.Tensor,
|
||||
kv_cache_dummy_dep: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
q, kv_c_normed, k_pe, output_attn, input_scale, kv_cache_dummy_dep
|
||||
):
|
||||
# MLA output in quant_dtype (FP4 packed as uint8)
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim // 2],
|
||||
@@ -207,7 +328,7 @@ class MLAAttnNvfp4QuantPattern(
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=self._layer_name,
|
||||
layer_name=_ln,
|
||||
output_scale=input_scale,
|
||||
output_block_scale=output_scale_view,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
@@ -217,7 +338,7 @@ class MLAAttnNvfp4QuantPattern(
|
||||
return _replacement
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
return [
|
||||
inputs: list = [
|
||||
self.empty(5, self._num_heads, self._qk_head_dim, dtype=self._dtype),
|
||||
self.empty(5, self._kv_lora_rank, dtype=self._dtype),
|
||||
self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype),
|
||||
@@ -225,6 +346,9 @@ class MLAAttnNvfp4QuantPattern(
|
||||
self.empty_fp32(1, 1),
|
||||
self.empty(0, dtype=self._dtype),
|
||||
]
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(_encode_layer_name(self._layer_name))
|
||||
return inputs
|
||||
|
||||
|
||||
class MLAAttnQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
@@ -250,13 +374,19 @@ class MLAAttnQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"so no fusion patterns were registered."
|
||||
)
|
||||
|
||||
# When _USE_LAYERNAME is enabled, layer_name is a wildcard so all
|
||||
# layers produce the same pattern — register once then break.
|
||||
for layer in layers:
|
||||
if layer.impl.fused_output_quant_supported(kFp8StaticTensorSym):
|
||||
self.register(MLAAttnFp8StaticQuantPattern(layer, dtype))
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"):
|
||||
for layer in layers:
|
||||
if layer.impl.fused_output_quant_supported(kNvfp4Dynamic):
|
||||
self.register(MLAAttnNvfp4QuantPattern(layer, dtype))
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
self.dump_patterns(config, self.pm_pass)
|
||||
|
||||
@@ -15,7 +15,13 @@ from vllm.model_executor.layers.attention.attention import (
|
||||
Attention,
|
||||
get_attention_context,
|
||||
)
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.utils.torch_utils import (
|
||||
_USE_LAYERNAME,
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
|
||||
from ..inductor_pass import enable_fake_mode
|
||||
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
|
||||
@@ -37,7 +43,7 @@ def fused_rope_and_unified_kv_cache_update_impl(
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
layer_name: str = "",
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This impl fetches the KV cache and slot mapping from the forward context,
|
||||
@@ -46,6 +52,7 @@ def fused_rope_and_unified_kv_cache_update_impl(
|
||||
that is passed to unified_attention to signal a side effect and
|
||||
the data dependency between them to ensure torch.compile preserves ordering.
|
||||
"""
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
_, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name)
|
||||
if layer_slot_mapping is not None:
|
||||
attn_layer.impl.do_rope_and_kv_cache_update(
|
||||
@@ -70,7 +77,7 @@ def fused_rope_and_unified_kv_cache_update_fake(
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
layer_name: str = "",
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(0, device=query.device, dtype=query.dtype)
|
||||
|
||||
@@ -120,38 +127,30 @@ class RopeReshapeKVCachePattern:
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
def get_inputs(self) -> list:
|
||||
# Sample inputs to help pattern tracing
|
||||
T = 5
|
||||
L = 4096
|
||||
qkv = empty_bf16(T, self.q_size + self.k_size + self.v_size)
|
||||
positions = empty_i64(T)
|
||||
cos_sin_cache = empty_bf16(L, self.head_size)
|
||||
return [
|
||||
qkv,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
]
|
||||
inputs: list = [qkv, positions, cos_sin_cache]
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(_encode_layer_name(self.layer_name))
|
||||
return inputs
|
||||
|
||||
def register(self, pm_pass: PatternMatcherPass) -> None:
|
||||
def pattern(
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
def _mk_pattern_with_layer_name_input(self, _ln):
|
||||
"""Pattern/replacement with layer_name as an explicit input."""
|
||||
|
||||
def pattern(qkv, positions, cos_sin_cache, layer_name):
|
||||
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
|
||||
q, k = self.rope_matcher(positions, q, k, cos_sin_cache)
|
||||
q = q.view(-1, self.num_heads, self.head_size)
|
||||
k = k.view(-1, self.num_kv_heads, self.head_size)
|
||||
v = v.view(-1, self.num_kv_heads, self.head_size_v)
|
||||
dummy = torch.ops.vllm.unified_kv_cache_update(k, v, self.layer_name)
|
||||
return dummy, q, k, v
|
||||
return torch.ops.vllm.unified_kv_cache_update(k, v, layer_name), q, k, v
|
||||
|
||||
def replacement(
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
def replacement(qkv, positions, cos_sin_cache, layer_name):
|
||||
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
|
||||
q = q.view(-1, self.num_heads, self.head_size)
|
||||
k = k.view(-1, self.num_kv_heads, self.head_size)
|
||||
@@ -164,10 +163,50 @@ class RopeReshapeKVCachePattern:
|
||||
positions=positions,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
is_neox=self.is_neox,
|
||||
layer_name=self.layer_name,
|
||||
layer_name=layer_name,
|
||||
)
|
||||
return results[0], results[1], results[2], v
|
||||
|
||||
return pattern, replacement
|
||||
|
||||
def _mk_pattern_with_layer_name_closure(self, _ln):
|
||||
"""Pattern/replacement with layer_name as a closure constant."""
|
||||
|
||||
def pattern(qkv, positions, cos_sin_cache):
|
||||
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
|
||||
q, k = self.rope_matcher(positions, q, k, cos_sin_cache)
|
||||
q = q.view(-1, self.num_heads, self.head_size)
|
||||
k = k.view(-1, self.num_kv_heads, self.head_size)
|
||||
v = v.view(-1, self.num_kv_heads, self.head_size_v)
|
||||
return torch.ops.vllm.unified_kv_cache_update(k, v, _ln), q, k, v
|
||||
|
||||
def replacement(qkv, positions, cos_sin_cache):
|
||||
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
|
||||
q = q.view(-1, self.num_heads, self.head_size)
|
||||
k = k.view(-1, self.num_kv_heads, self.head_size)
|
||||
v = v.view(-1, self.num_kv_heads, self.head_size_v)
|
||||
results = auto_functionalized(
|
||||
self.FUSED_OP,
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
positions=positions,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
is_neox=self.is_neox,
|
||||
layer_name=_ln,
|
||||
)
|
||||
return results[0], results[1], results[2], v
|
||||
|
||||
return pattern, replacement
|
||||
|
||||
def register(self, pm_pass: PatternMatcherPass) -> None:
|
||||
_ln = _encode_layer_name(self.layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
pattern, replacement = self._mk_pattern_with_layer_name_input(_ln)
|
||||
else:
|
||||
pattern, replacement = self._mk_pattern_with_layer_name_closure(_ln)
|
||||
|
||||
# NOTE: use view_to_reshape to unify view/reshape to simplify
|
||||
# pattern and increase matching opportunities
|
||||
def fwd_and_view_to_reshape(*args, **kwargs) -> fx.GraphModule:
|
||||
@@ -176,7 +215,11 @@ class RopeReshapeKVCachePattern:
|
||||
return gm
|
||||
|
||||
pm.register_replacement(
|
||||
pattern, replacement, self.get_inputs(), fwd_and_view_to_reshape, pm_pass
|
||||
pattern,
|
||||
replacement,
|
||||
self.get_inputs(),
|
||||
fwd_and_view_to_reshape,
|
||||
pm_pass,
|
||||
)
|
||||
|
||||
|
||||
@@ -205,6 +248,8 @@ class RopeKVCacheFusionPass(VllmPatternMatcherPass):
|
||||
self.max_token_num = cc.pass_config.rope_kvcache_fusion_max_token_num
|
||||
|
||||
attn_layers = get_layers_from_vllm_config(config, Attention)
|
||||
# When _USE_LAYERNAME is enabled, layer_name is a wildcard so all
|
||||
# layers produce the same pattern — register once then break.
|
||||
for _, layer in attn_layers.items():
|
||||
if layer.impl.fused_rope_kvcache_supported():
|
||||
for is_neox in [True, False]:
|
||||
@@ -212,6 +257,8 @@ class RopeKVCacheFusionPass(VllmPatternMatcherPass):
|
||||
layer=layer,
|
||||
is_neox=is_neox,
|
||||
).register(self.patterns)
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
self.dump_patterns(config, self.patterns)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user