[Bugfix] moe lora align kernel grid (#40131)
Signed-off-by: TheDuyIT <nduy250299@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: dtnguyen <dtnguyen@nvidia.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
Jee Jee Li
Jee Jee Li
parent
c1f7854342
commit
e3aeee5ff8
@@ -104,5 +104,222 @@ def test_moe_lora_align_block_size(
|
||||
assert torch.all(topk_ids.view(-1)[indices] == expert_id)
|
||||
|
||||
|
||||
# Sentinel values for the regression tests below. Distinctive out-of-domain
|
||||
# ints so that "kernel never wrote this slot" is directly observable: the
|
||||
# kernel only ever writes a real expert id in [0, num_experts) or -1
|
||||
# (expert_ids), a token index or the `numel` padding value (sorted_token_ids),
|
||||
# and a block-aligned cumsum count (num_tokens_post_pad).
|
||||
SENTINEL_EXPERT = -2
|
||||
SENTINEL_TOKEN = -7
|
||||
SENTINEL_NPAD = -13
|
||||
|
||||
|
||||
def _build_and_run_align(
|
||||
*,
|
||||
num_lora_tokens,
|
||||
num_base_tokens,
|
||||
max_loras,
|
||||
num_experts=64,
|
||||
topk_num=6,
|
||||
block_size=16,
|
||||
lora_ids_override=None,
|
||||
disabled_slots=(),
|
||||
seed=1,
|
||||
):
|
||||
"""Build inputs the way ``LoRAKernelMeta.prepare_tensors`` does, run
|
||||
``moe_lora_align_block_size``, and return a dict of result tensors plus
|
||||
derived sizes. Output buffers are pre-filled with ``SENTINEL_*`` so
|
||||
callers can assert which slots the kernel did / did not touch.
|
||||
|
||||
Tokens are assigned to LoRA slot 0 (first ``num_lora_tokens``) then -1
|
||||
(remaining ``num_base_tokens``), matching the "mixed base + 1 LoRA"
|
||||
shape used to repro vllm-project/vllm#32235.
|
||||
|
||||
``lora_ids_override``: optional 1-D int tensor of length ``max_loras+1``
|
||||
used verbatim. Default mirrors ``prepare_tensors`` (sorted-unique into
|
||||
the head, -1 tail).
|
||||
``disabled_slots``: iterable of slot indices to clear in ``adapter_enabled``.
|
||||
"""
|
||||
random.seed(seed)
|
||||
num_tokens = num_lora_tokens + num_base_tokens
|
||||
assert num_tokens > 0, "test requires at least one token"
|
||||
|
||||
topk_ids = torch.zeros((num_tokens, topk_num), dtype=torch.int32)
|
||||
token_lora_mapping = torch.empty((num_tokens,), dtype=torch.int32)
|
||||
for i in range(num_tokens):
|
||||
pool = list(range(num_experts))
|
||||
random.shuffle(pool)
|
||||
for j in range(topk_num):
|
||||
topk_ids[i, j] = pool[j]
|
||||
token_lora_mapping[i] = 0 if i < num_lora_tokens else -1
|
||||
topk_ids = topk_ids.to(DEVICE_TYPE)
|
||||
token_lora_mapping = token_lora_mapping.to(DEVICE_TYPE)
|
||||
|
||||
max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1)
|
||||
max_num_tokens_padded = round_up(max_num_tokens_padded, block_size)
|
||||
if topk_ids.numel() < num_experts:
|
||||
max_num_tokens_padded = topk_ids.numel() * block_size
|
||||
max_num_m_blocks = CEILDIV(max_num_tokens_padded, block_size)
|
||||
|
||||
if lora_ids_override is None:
|
||||
lora_ids = torch.full(
|
||||
(max_loras + 1,), -1, dtype=torch.int32, device=DEVICE_TYPE
|
||||
)
|
||||
unique_ids = torch.unique(token_lora_mapping, sorted=True)
|
||||
lora_ids[: unique_ids.numel()] = unique_ids.to(torch.int32)
|
||||
else:
|
||||
assert lora_ids_override.numel() == max_loras + 1
|
||||
lora_ids = lora_ids_override.to(dtype=torch.int32, device=DEVICE_TYPE)
|
||||
|
||||
adapter_enabled = torch.ones(
|
||||
(max_loras + 1,), dtype=torch.int32, device=DEVICE_TYPE
|
||||
)
|
||||
for slot in disabled_slots:
|
||||
adapter_enabled[slot] = 0
|
||||
|
||||
sorted_token_ids = torch.full(
|
||||
(max_loras * max_num_tokens_padded,),
|
||||
SENTINEL_TOKEN,
|
||||
dtype=torch.int32,
|
||||
device=DEVICE_TYPE,
|
||||
)
|
||||
expert_ids = torch.full(
|
||||
(max_loras * max_num_m_blocks,),
|
||||
SENTINEL_EXPERT,
|
||||
dtype=torch.int32,
|
||||
device=DEVICE_TYPE,
|
||||
)
|
||||
num_tokens_post_pad = torch.full(
|
||||
(max_loras,), SENTINEL_NPAD, dtype=torch.int32, device=DEVICE_TYPE
|
||||
)
|
||||
|
||||
ops.moe_lora_align_block_size(
|
||||
topk_ids,
|
||||
token_lora_mapping,
|
||||
num_experts,
|
||||
block_size,
|
||||
max_loras,
|
||||
max_num_tokens_padded,
|
||||
max_num_m_blocks,
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_pad,
|
||||
adapter_enabled,
|
||||
lora_ids,
|
||||
)
|
||||
|
||||
return {
|
||||
"lora_ids": lora_ids,
|
||||
"sorted_token_ids": sorted_token_ids,
|
||||
"expert_ids": expert_ids,
|
||||
"num_tokens_post_pad": num_tokens_post_pad,
|
||||
"max_num_tokens_padded": max_num_tokens_padded,
|
||||
"block_size": block_size,
|
||||
"max_loras": max_loras,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"max_loras",
|
||||
[
|
||||
1,
|
||||
2,
|
||||
],
|
||||
)
|
||||
def test_moe_lora_align_block_size_mixed_base_and_lora(max_loras):
|
||||
"""Regression test for issue #32235: real LoRA slot must not be skipped
|
||||
when ``active_lora_ids`` has -1 at position 0."""
|
||||
out = _build_and_run_align(
|
||||
num_lora_tokens=8, num_base_tokens=8, max_loras=max_loras
|
||||
)
|
||||
|
||||
# Sanity check on the layout being tested.
|
||||
assert out["lora_ids"][0].item() == -1, (
|
||||
"prepare_tensors layout mismatch: -1 expected at position 0 for mixed batch"
|
||||
)
|
||||
|
||||
real_slot = 0
|
||||
post_pad = out["num_tokens_post_pad"][real_slot].item()
|
||||
assert post_pad != SENTINEL_NPAD, (
|
||||
f"num_tokens_post_pad[{real_slot}] was never written by the kernel; "
|
||||
"the align kernel skipped the real LoRA slot."
|
||||
)
|
||||
assert (
|
||||
0 < post_pad <= out["max_num_tokens_padded"]
|
||||
and post_pad % out["block_size"] == 0
|
||||
), f"num_tokens_post_pad[{real_slot}]={post_pad} is not a valid block-aligned count"
|
||||
|
||||
expert_row = out["expert_ids"].view(max_loras, -1)[real_slot]
|
||||
assert (expert_row != SENTINEL_EXPERT).all(), (
|
||||
f"expert_ids row for slot {real_slot} has unwritten sentinel entries; "
|
||||
"the align kernel skipped the real LoRA slot."
|
||||
)
|
||||
|
||||
sorted_row = out["sorted_token_ids"].view(max_loras, -1)[real_slot]
|
||||
assert (sorted_row != SENTINEL_TOKEN).all(), (
|
||||
f"sorted_token_ids row for slot {real_slot} has unwritten sentinel "
|
||||
"entries; the align kernel skipped the real LoRA slot."
|
||||
)
|
||||
|
||||
|
||||
def test_moe_lora_align_block_size_disabled_adapter_untouched():
|
||||
"""Disabled-adapter slot rows must remain untouched by all three align
|
||||
kernels. Pins the invariant protected by the ``adapter_enabled`` guard
|
||||
in ``lora_count_and_sort_expert_tokens_kernel``: without it the sort
|
||||
kernel reads uninitialized ``token_mask`` values for disabled slots and
|
||||
pollutes ``sorted_token_ids`` / ``cumsum_buffer``."""
|
||||
max_loras = 1
|
||||
out = _build_and_run_align(
|
||||
num_lora_tokens=16,
|
||||
num_base_tokens=0,
|
||||
max_loras=max_loras,
|
||||
disabled_slots=(0,),
|
||||
)
|
||||
# Sanity: slot 0 IS present in active_lora_ids (otherwise we would only
|
||||
# exercise the lora_id == -1 / >= max_loras guards).
|
||||
assert (out["lora_ids"] == 0).any().item()
|
||||
|
||||
assert out["num_tokens_post_pad"][0].item() == SENTINEL_NPAD, (
|
||||
"num_tokens_post_pad[0] was modified for a disabled adapter slot."
|
||||
)
|
||||
expert_row = out["expert_ids"].view(max_loras, -1)[0]
|
||||
assert (expert_row == SENTINEL_EXPERT).all(), (
|
||||
"expert_ids row for disabled slot 0 was partially written."
|
||||
)
|
||||
# Row specifically protected by the sort-kernel adapter_enabled guard.
|
||||
sorted_row = out["sorted_token_ids"].view(max_loras, -1)[0]
|
||||
assert (sorted_row == SENTINEL_TOKEN).all(), (
|
||||
"sorted_token_ids row for disabled slot 0 was polluted by the sort "
|
||||
"kernel; lora_count_and_sort_expert_tokens_kernel must skip "
|
||||
"adapter_enabled == 0 slots."
|
||||
)
|
||||
|
||||
|
||||
def test_moe_lora_align_block_size_lora_id_oob_guard():
|
||||
"""Regression test for the ``lora_id >= max_loras`` guard.
|
||||
|
||||
Production ``LoRAKernelMeta.prepare_tensors`` pre-fills the tail of
|
||||
``active_lora_ids`` with -1, so the existing ``lora_id == -1`` check
|
||||
covers the extra slot. This test bypasses that invariant and injects
|
||||
an out-of-range value (5 with max_loras=1) at the tail to verify the
|
||||
explicit guard prevents OOB reads against ``adapter_enabled`` and
|
||||
OOB writes against the max_loras-sized output buffers. Without the
|
||||
guard, an illegal-memory-access would surface on the next CUDA sync.
|
||||
"""
|
||||
max_loras = 1
|
||||
lora_ids_override = torch.tensor([0, 5], dtype=torch.int32)
|
||||
out = _build_and_run_align(
|
||||
num_lora_tokens=16,
|
||||
num_base_tokens=0,
|
||||
max_loras=max_loras,
|
||||
lora_ids_override=lora_ids_override,
|
||||
)
|
||||
# The .item() call below syncs and would surface any async
|
||||
# illegal-memory-access from the OOB iteration.
|
||||
assert out["num_tokens_post_pad"][0].item() != SENTINEL_NPAD, (
|
||||
"real LoRA slot 0 was skipped by the align kernel"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__])
|
||||
|
||||
Reference in New Issue
Block a user