Eliminate redundant MoE buffer copies in AITER fused experts (without dependency on AITER changes) (#41713)
Signed-off-by: Mehdi Ghanimifard <mehdi.ghanimifard@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
6e6d182d18
commit
d4b0048404
@@ -1215,6 +1215,7 @@ class FusedMoEKernelModularImpl:
|
|||||||
expert_map: torch.Tensor | None,
|
expert_map: torch.Tensor | None,
|
||||||
apply_router_weight_on_input: bool,
|
apply_router_weight_on_input: bool,
|
||||||
expert_tokens_meta: ExpertTokensMetadata | None,
|
expert_tokens_meta: ExpertTokensMetadata | None,
|
||||||
|
output_alias: torch.Tensor | None = None,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
_, M_full, N, K, top_k = self.fused_experts.moe_problem_size(
|
_, M_full, N, K, top_k = self.fused_experts.moe_problem_size(
|
||||||
a1q, w1, w2, topk_ids
|
a1q, w1, w2, topk_ids
|
||||||
@@ -1243,6 +1244,23 @@ class FusedMoEKernelModularImpl:
|
|||||||
activation,
|
activation,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# If caller's output buffer already matches fused_out shape/dtype, alias
|
||||||
|
# to skip the redundant copy in TopKWeightAndReduceNoOP.apply downstream.
|
||||||
|
# This eliminates ~94% of __amd_rocclr_copyBuffer events (Copy 2 of the
|
||||||
|
# double-copy MoE write-back path).
|
||||||
|
if current_platform.is_rocm():
|
||||||
|
from vllm._aiter_ops import rocm_aiter_ops
|
||||||
|
|
||||||
|
if (
|
||||||
|
rocm_aiter_ops.is_fused_moe_enabled()
|
||||||
|
and output_alias is not None
|
||||||
|
and output_alias.shape == fused_out.shape
|
||||||
|
and output_alias.dtype == fused_out.dtype
|
||||||
|
and output_alias.device == fused_out.device
|
||||||
|
and output_alias.is_contiguous()
|
||||||
|
):
|
||||||
|
fused_out = output_alias
|
||||||
|
|
||||||
self.fused_experts.apply(
|
self.fused_experts.apply(
|
||||||
output=fused_out,
|
output=fused_out,
|
||||||
hidden_states=a1q,
|
hidden_states=a1q,
|
||||||
@@ -1403,6 +1421,7 @@ class FusedMoEKernelModularImpl:
|
|||||||
expert_map=expert_map,
|
expert_map=expert_map,
|
||||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||||
expert_tokens_meta=expert_tokens_meta,
|
expert_tokens_meta=expert_tokens_meta,
|
||||||
|
output_alias=output,
|
||||||
)
|
)
|
||||||
|
|
||||||
return self._finalize(
|
return self._finalize(
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
|||||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||||
TopKWeightAndReduceNoOP,
|
TopKWeightAndReduceNoOP,
|
||||||
)
|
)
|
||||||
|
from vllm.model_executor.layers.fused_moe.utils import disable_inplace
|
||||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||||
QuantKey,
|
QuantKey,
|
||||||
kFp8Dynamic128Sym,
|
kFp8Dynamic128Sym,
|
||||||
@@ -438,4 +439,16 @@ class AiterExperts(mk.FusedMoEExpertsModular):
|
|||||||
num_local_tokens=num_local_tokens,
|
num_local_tokens=num_local_tokens,
|
||||||
output_dtype=output.dtype,
|
output_dtype=output.dtype,
|
||||||
)
|
)
|
||||||
output.copy_(result)
|
# avoid redundant copy when output is a view of the result
|
||||||
|
if (
|
||||||
|
output.shape == result.shape
|
||||||
|
and output.dtype == result.dtype
|
||||||
|
and output.device == result.device
|
||||||
|
and output.is_contiguous()
|
||||||
|
and result.is_contiguous()
|
||||||
|
and output._base is None
|
||||||
|
and disable_inplace()
|
||||||
|
):
|
||||||
|
output.set_(result)
|
||||||
|
else:
|
||||||
|
output.copy_(result)
|
||||||
|
|||||||
@@ -62,6 +62,10 @@ class TopKWeightAndReduceNoOP(mk.TopKWeightAndReduce):
|
|||||||
if output is None:
|
if output is None:
|
||||||
return fused_expert_output
|
return fused_expert_output
|
||||||
|
|
||||||
|
# Skip self-copy when caller aliased fused_out to output upstream.
|
||||||
|
if output is fused_expert_output:
|
||||||
|
return output
|
||||||
|
|
||||||
# MoEPrepareAndFinalizeNoDPEPModular needs the output to be in the `output`
|
# MoEPrepareAndFinalizeNoDPEPModular needs the output to be in the `output`
|
||||||
# tensor.
|
# tensor.
|
||||||
assert output.size() == fused_expert_output.size(), (
|
assert output.size() == fused_expert_output.size(), (
|
||||||
|
|||||||
Reference in New Issue
Block a user