buffer size insuffient Dspark sd for FlashInfer MNNVL allreduce (#50932)
This commit is contained in:
@@ -386,7 +386,6 @@ class FlashInferAllReduce:
|
||||
self.world_size,
|
||||
)
|
||||
return
|
||||
|
||||
backend, _ = _resolve_fi_ar_backend()
|
||||
tuned_max_size = _get_tuned_standalone_max_size(
|
||||
self.world_size,
|
||||
@@ -450,7 +449,27 @@ class FlashInferAllReduce:
|
||||
if num_tokens > self.max_num_tokens:
|
||||
return False
|
||||
|
||||
return self._ensure_workspace(hidden_dim, input_tensor.dtype)
|
||||
if not self._ensure_workspace(hidden_dim, input_tensor.dtype):
|
||||
return False
|
||||
|
||||
workspace = get_fi_ar_workspace(
|
||||
world_size=self.world_size,
|
||||
rank=self.rank,
|
||||
max_token_num=self.max_num_tokens,
|
||||
hidden_dim=hidden_dim,
|
||||
dtype=input_tensor.dtype,
|
||||
group=self.group,
|
||||
)
|
||||
assert workspace is not None
|
||||
# The token bound above uses the full allocation budget, but mnnvl's
|
||||
# Lamport buffers rotate through three slots, so only ~1/3 is usable per
|
||||
# call. Asking the workspace directly to reject sizes the kernel can't fit.
|
||||
return workspace.is_buffer_size_sufficient(
|
||||
tp_size=self.world_size,
|
||||
num_tokens=num_tokens,
|
||||
hidden_dim=hidden_dim,
|
||||
dtype=input_tensor.dtype,
|
||||
)
|
||||
|
||||
def all_reduce(self, input_tensor: torch.Tensor) -> torch.Tensor:
|
||||
num_tokens, hidden_dim = input_tensor.shape
|
||||
|
||||
@@ -97,6 +97,17 @@ def _can_use_flashinfer(hidden_states: torch.Tensor, tp_size: int) -> tuple[bool
|
||||
)
|
||||
if workspace is None:
|
||||
return False, 0
|
||||
# The token-count bound above uses the whole workspace budget, but a backend
|
||||
# may use only a fraction of it per call (mnnvl rotates through three Lamport
|
||||
# buffers). Ask the workspace so we don't admit tensors that the kernel will
|
||||
# reject with "The buffer size in the given workspace is insufficient".
|
||||
if not workspace.is_buffer_size_sufficient(
|
||||
tp_size=tp_size,
|
||||
num_tokens=num_tokens,
|
||||
hidden_dim=hidden_size,
|
||||
dtype=hidden_states.dtype,
|
||||
):
|
||||
return False, 0
|
||||
return True, max_token_num
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user