[NIXL][BUG] Fix Triton heterogeneous TP (#37940)
Signed-off-by: Yifan <yzong@redhat.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
parent
148c2072ec
commit
dc0428ebb8
@@ -23,6 +23,10 @@ hybrid_ssm_configs=(
|
||||
# TODO: (NickLucche) Address async scheduling issue with TP>1 separately as this may impact other models.
|
||||
"ENABLE_HMA_FLAG=1 PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ibm-granite/granite-4.0-h-tiny VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192,--trust-remote-code,--no-async-scheduling"
|
||||
)
|
||||
sw_attn_configs=(
|
||||
"ENABLE_HMA_FLAG=1 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-3-4b-it PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192"
|
||||
"ENABLE_HMA_FLAG=1 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-3-4b-it PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=1 VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192"
|
||||
)
|
||||
|
||||
# Select config array based on DP_EP env var
|
||||
if [[ -n "${DP_EP:-}" ]]; then
|
||||
@@ -31,6 +35,9 @@ if [[ -n "${DP_EP:-}" ]]; then
|
||||
elif [[ -n "${HYBRID_SSM:-}" ]]; then
|
||||
configs=("${hybrid_ssm_configs[@]}")
|
||||
echo "HYBRID_SSM is set, using hybrid_ssm_configs."
|
||||
elif [[ -n "${SW_ATTN:-}" ]]; then
|
||||
configs=("${sw_attn_configs[@]}")
|
||||
echo "SW_ATTN is set, using sw_attn_configs."
|
||||
else
|
||||
configs=("${tp_configs[@]}")
|
||||
fi
|
||||
|
||||
@@ -1586,10 +1586,18 @@ def test_register_kv_caches(
|
||||
expected_base_addrs: list[int]
|
||||
expected_num_entries: int
|
||||
kv_caches: dict[str, torch.Tensor]
|
||||
assert str(enable_cross_layers).lower() != "true" or (
|
||||
(attn_backend not in ("FLASH_ATTN", "FLASHINFER"))
|
||||
or connector.prefer_cross_layer_blocks
|
||||
if str(enable_cross_layers).lower() == "true":
|
||||
assert connector.prefer_cross_layer_blocks == (
|
||||
attn_backend in ("FLASH_ATTN", "FLASHINFER", "TRITON_ATTN")
|
||||
)
|
||||
else:
|
||||
assert not connector.prefer_cross_layer_blocks
|
||||
|
||||
test_shape = backend_cls.get_kv_cache_shape(
|
||||
num_blocks=1, block_size=16, num_kv_heads=1, head_size=1
|
||||
)
|
||||
is_blocks_first = len(test_shape) == 5 and test_shape[0] == 1
|
||||
|
||||
if connector.prefer_cross_layer_blocks:
|
||||
with set_current_vllm_config(vllm_config):
|
||||
_, cross_layers_kv_cache, _ = (
|
||||
@@ -1619,7 +1627,7 @@ def test_register_kv_caches(
|
||||
]
|
||||
expected_num_entries = 1
|
||||
|
||||
expected_blocks_count = 8
|
||||
expected_blocks_count = num_blocks * (2 if is_blocks_first else 1)
|
||||
|
||||
kv_caches = {"all-layers": cross_layers_kv_cache}
|
||||
else:
|
||||
@@ -1639,12 +1647,6 @@ def test_register_kv_caches(
|
||||
}
|
||||
|
||||
# Store tensor info for validation
|
||||
|
||||
test_shape = backend_cls.get_kv_cache_shape(
|
||||
num_blocks=1, block_size=16, num_kv_heads=1, head_size=1
|
||||
)
|
||||
is_blocks_first = len(test_shape) == 5 and test_shape[0] == 1
|
||||
|
||||
if is_blocks_first:
|
||||
expected_tensor_size = (
|
||||
shared_tensor.element_size() * shared_tensor.numel()
|
||||
@@ -1696,13 +1698,13 @@ def test_register_kv_caches(
|
||||
|
||||
if connector.prefer_cross_layer_blocks:
|
||||
num_blocks = 8
|
||||
expected_block_len = expected_tensor_size // num_blocks
|
||||
else:
|
||||
num_blocks = kv_cache_config.num_blocks
|
||||
if is_blocks_first:
|
||||
expected_block_len = expected_tensor_size // num_blocks // 2
|
||||
else:
|
||||
expected_block_len = expected_tensor_size // num_blocks
|
||||
|
||||
if is_blocks_first:
|
||||
expected_block_len = expected_tensor_size // num_blocks // 2
|
||||
else:
|
||||
expected_block_len = expected_tensor_size // num_blocks
|
||||
|
||||
for i, block_entry in enumerate(blocks_data):
|
||||
block_start_addr, block_len, tp_rank = block_entry
|
||||
|
||||
Reference in New Issue
Block a user