[KVConnector][NIXL] Support pipeline-parallel prefill in push mode (#45880)
Signed-off-by: zixi-qi <zixi@inferact.ai> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -40,7 +40,6 @@ sw_attn_configs=(
|
||||
# Gemma4: SW + cross-layer KV sharing
|
||||
"GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-4-E2B-it VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192"
|
||||
)
|
||||
|
||||
# Select config array based on DP_EP env var
|
||||
if [[ -n "${DP_EP:-}" ]]; then
|
||||
configs=("${dp_ep_configs[@]}")
|
||||
|
||||
@@ -15,6 +15,7 @@ def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--host", required=True)
|
||||
parser.add_argument("--port", required=True, type=int)
|
||||
parser.add_argument("--pp-rank", default=0, type=int)
|
||||
parser.add_argument("--rank", default=0, type=int)
|
||||
parser.add_argument("--timeout-ms", default=1000, type=int)
|
||||
return parser.parse_args()
|
||||
@@ -37,7 +38,7 @@ def main() -> None:
|
||||
sock.setsockopt(zmq.RCVTIMEO, args.timeout_ms)
|
||||
try:
|
||||
sock.connect(make_zmq_path(args.host, args.port))
|
||||
sock.send(msgspec.msgpack.encode((GET_META_MSG, args.rank)))
|
||||
sock.send(msgspec.msgpack.encode((GET_META_MSG, args.pp_rank, args.rank)))
|
||||
sock.recv()
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
@@ -49,13 +49,16 @@ else
|
||||
KV_EXTRA_CONFIG=''
|
||||
fi
|
||||
|
||||
# Connector: default pull NixlConnector; NixlPushConnector enables PP prefill.
|
||||
KV_CONNECTOR=${KV_CONNECTOR:-NixlConnector}
|
||||
|
||||
# Build the kv-transfer-config for P and D
|
||||
if [[ "$KV_BUFFER_DEVICE" == "cuda" ]]; then
|
||||
KV_CONFIG_P='{"kv_connector":"NixlConnector","kv_role":"kv_producer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'
|
||||
KV_CONFIG_D='{"kv_connector":"NixlConnector","kv_role":"kv_consumer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'
|
||||
KV_CONFIG_P='{"kv_connector":"'"$KV_CONNECTOR"'","kv_role":"kv_producer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'
|
||||
KV_CONFIG_D='{"kv_connector":"'"$KV_CONNECTOR"'","kv_role":"kv_consumer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'
|
||||
else
|
||||
KV_CONFIG_P="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_producer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"
|
||||
KV_CONFIG_D="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_consumer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"
|
||||
KV_CONFIG_P="{\"kv_connector\":\"$KV_CONNECTOR\",\"kv_role\":\"kv_producer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"
|
||||
KV_CONFIG_D="{\"kv_connector\":\"$KV_CONNECTOR\",\"kv_role\":\"kv_consumer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"
|
||||
fi
|
||||
|
||||
# Models to run
|
||||
@@ -72,6 +75,7 @@ fi
|
||||
NUM_PREFILL_INSTANCES=${NUM_PREFILL_INSTANCES:-1} # Default to 1
|
||||
NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1} # Default to 1
|
||||
PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1}
|
||||
PREFILLER_PP_SIZE=${PREFILLER_PP_SIZE:-1} # >1 requires NixlPushConnector
|
||||
DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
|
||||
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.2}
|
||||
PREFILL_BLOCK_SIZE=${PREFILL_BLOCK_SIZE:-128}
|
||||
@@ -138,8 +142,9 @@ run_tests_for_model() {
|
||||
# Calculate GPU ID - we'll distribute across available GPUs
|
||||
GPU_ID=$((i % $(get_num_gpus)))
|
||||
NEXT_GPU=${GPU_ID}
|
||||
# If PREFILLER_TP_SIZE is more than 1
|
||||
for (( j=1; j < PREFILLER_TP_SIZE; j++ )); do
|
||||
# Reserve TP*PP GPUs for the prefiller (TP shards across PP stages).
|
||||
PREFILLER_WORLD_SIZE=$((PREFILLER_TP_SIZE * PREFILLER_PP_SIZE))
|
||||
for (( j=1; j < PREFILLER_WORLD_SIZE; j++ )); do
|
||||
NEXT_GPU=$(((GPU_ID + j) % $(get_num_gpus)))
|
||||
GPU_ID="${GPU_ID},${NEXT_GPU}"
|
||||
done
|
||||
@@ -161,6 +166,7 @@ run_tests_for_model() {
|
||||
--block-size ${PREFILL_BLOCK_SIZE} \
|
||||
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
||||
--tensor-parallel-size $PREFILLER_TP_SIZE \
|
||||
--pipeline-parallel-size $PREFILLER_PP_SIZE \
|
||||
--kv-transfer-config '$KV_CONFIG_P'"
|
||||
if [[ "$ENFORCE_EAGER" == "1" ]]; then
|
||||
BASE_CMD="${BASE_CMD} --enforce-eager"
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# NixlPushConnector (push-mode PD) integration tests. Kept separate from the
|
||||
# pull-mode sweep in ../nixl_integration so push coverage can grow on its own;
|
||||
# the per-config runner is shared.
|
||||
SCRIPT="v1/kv_connector/nixl_integration/run_accuracy_test.sh"
|
||||
IMPORT_CANARY="v1/kv_connector/nixl_integration/test_nixl_imports.py"
|
||||
|
||||
echo "=== Running NIXL import canary ==="
|
||||
python3 -m pytest -s -x "${IMPORT_CANARY}"
|
||||
|
||||
# PP=2 prefill -> TP=2 decode: each PP stage writes its layer slice into the
|
||||
# decoder's matching region range (non-MLA and MLA contiguous-slice paths).
|
||||
configs=(
|
||||
"KV_CONNECTOR=NixlPushConnector GPU_MEMORY_UTILIZATION=0.6 PREFILLER_PP_SIZE=2 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2" # non-MLA, Qwen3-0.6B
|
||||
"KV_CONNECTOR=NixlPushConnector GPU_MEMORY_UTILIZATION=0.8 PREFILLER_PP_SIZE=2 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 MODEL_NAMES=deepseek-ai/deepseek-vl2-tiny" # MLA
|
||||
)
|
||||
|
||||
run_tests() {
|
||||
local label=$1
|
||||
local extra_args=$2
|
||||
|
||||
echo "=== Running push tests (${label}) ==="
|
||||
for cfg in "${configs[@]}"; do
|
||||
local -a cfg_parts extra_args_parts
|
||||
read -r -a cfg_parts <<< "$cfg"
|
||||
read -r -a extra_args_parts <<< "$extra_args"
|
||||
|
||||
echo "-> Running with ${cfg} ${extra_args:+and ${extra_args}}"
|
||||
# Use 'env' to safely set variables without eval; keep argv splitting
|
||||
# safe and SC2086-clean via arrays.
|
||||
if ! env "${cfg_parts[@]}" bash "${SCRIPT}" "${extra_args_parts[@]}"; then
|
||||
echo "❌ Test failed for config: ${cfg} ${extra_args:+(${extra_args})}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "✅ All ${label} push tests passed!"
|
||||
}
|
||||
|
||||
run_tests "default backend" ""
|
||||
@@ -408,11 +408,10 @@ def test_kv_transfer_handshake(dist_init):
|
||||
decoder = msgspec.msgpack.Decoder(NixlAgentMetadata)
|
||||
expected_agent_metadata = decoder.decode(metadata.agent_metadata_bytes)
|
||||
|
||||
# The scheduler connector expects metadata to be in
|
||||
# dict[int, KVConnectorHandshakeMetadata], where the first key is
|
||||
# the dp_rank, the second key is the tp_rank.
|
||||
# The scheduler connector expects metadata keyed by
|
||||
# (pp_rank, tp_rank).
|
||||
scheduler_connector = scheduler.get_kv_connector()
|
||||
scheduler_connector.set_xfer_handshake_metadata({0: metadata})
|
||||
scheduler_connector.set_xfer_handshake_metadata_pp_aware({(0, 0): metadata})
|
||||
|
||||
# Simulate a request that finishes prefill, which returns
|
||||
# corresponding NixlConnectorMetadata for decode instance.
|
||||
@@ -502,8 +501,14 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
|
||||
)
|
||||
|
||||
def _nixl_handshake(
|
||||
self, host: str, port: int, remote_tp_size: int, expected_engine_id: str
|
||||
) -> dict[int, str]:
|
||||
self,
|
||||
host: str,
|
||||
port: int,
|
||||
remote_tp_size: int,
|
||||
expected_engine_id: str,
|
||||
remote_pp_size: int = 1,
|
||||
notif_agents_only: bool = False,
|
||||
) -> dict[tuple[int, int], str]:
|
||||
# Mimic slow _nixl_handshake, as well as bypass zmq communication.
|
||||
time.sleep(self._hand_shake_latency)
|
||||
# These should've been done in register_kv_caches(), called by
|
||||
@@ -533,7 +538,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
|
||||
# When remote tp_size > local tp_size, handshake with multiple
|
||||
# remote ranks.
|
||||
num_handshakes = 1 if tp_ratio > 0 else -tp_ratio
|
||||
remote_agents: dict[int, str] = {}
|
||||
remote_agents: dict[tuple[int, int], str] = {}
|
||||
for remote_tp_rank in range(num_handshakes):
|
||||
remote_agent_name = self.add_remote_agent(
|
||||
NixlAgentMetadata(
|
||||
@@ -554,7 +559,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
|
||||
remote_tp_rank=remote_tp_rank,
|
||||
remote_tp_size=remote_tp_size,
|
||||
)
|
||||
remote_agents[remote_tp_rank] = remote_agent_name
|
||||
remote_agents[(0, remote_tp_rank)] = remote_agent_name
|
||||
return remote_agents
|
||||
|
||||
|
||||
@@ -749,7 +754,7 @@ class TestNixlHandshake:
|
||||
|
||||
def check_handshake(remote_tp_size: int):
|
||||
tp_ratio = remote_tp_size // local_tp_size
|
||||
assert set(remote_agents.keys()) == set(range(tp_ratio))
|
||||
assert set(remote_agents.keys()) == {(0, r) for r in range(tp_ratio)}
|
||||
|
||||
remote_engine_id = worker.REMOTE_ENGINE_ID
|
||||
remote_info = worker.transfer_topo.get_engine_info(remote_engine_id)
|
||||
@@ -2044,7 +2049,7 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init):
|
||||
# P TP = 2 * D TP case, we should register 2 local handles
|
||||
worker.src_xfer_handles_by_tp_ratio = {-2: [456, 457]}
|
||||
worker.dst_xfer_side_handles = {"engine1": {0: 789}}
|
||||
worker._remote_agents = {"engine1": {0: "agent1"}}
|
||||
worker._remote_agents = {"engine1": {(0, 0): "agent1"}}
|
||||
# _cleanup_remote_engine (called by shutdown) also clears these:
|
||||
worker.kv_caches_base_addr["engine1"] = {0: [0xABC]}
|
||||
worker.dst_num_blocks["engine1"] = 50
|
||||
@@ -2097,7 +2102,7 @@ def _setup_worker_with_remote_engine(
|
||||
)
|
||||
|
||||
engine_id = "remote-engine-1"
|
||||
worker._remote_agents[engine_id] = {0: "agent_0", 1: "agent_1"}
|
||||
worker._remote_agents[engine_id] = {(0, 0): "agent_0", (0, 1): "agent_1"}
|
||||
worker.dst_xfer_side_handles[engine_id] = {0: 100, 1: 200}
|
||||
worker.kv_caches_base_addr[engine_id] = {0: [0xABC]}
|
||||
worker.dst_num_blocks[engine_id] = 50
|
||||
@@ -2993,7 +2998,7 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario)
|
||||
local_block_len=worker.block_size * 4096,
|
||||
)
|
||||
worker._remote_agents[remote_engine_id] = {
|
||||
rank: f"agent_p{rank}" for rank in range(prefill_tp_size)
|
||||
(0, rank): f"agent_p{rank}" for rank in range(prefill_tp_size)
|
||||
}
|
||||
worker.dst_xfer_side_handles = {
|
||||
remote_engine_id: {rank: 100 + rank for rank in range(prefill_tp_size)}
|
||||
@@ -3044,7 +3049,7 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario)
|
||||
|
||||
# Broadcast goes to ranks {1, 2, 3} only, never to the read target.
|
||||
expected_recipients = {
|
||||
worker._remote_agents[remote_engine_id][r]
|
||||
worker._remote_agents[remote_engine_id][(0, r)]
|
||||
for r in range(1, prefill_tp_size)
|
||||
}
|
||||
assert {agent for agent, _ in send_notif_calls} == expected_recipients
|
||||
|
||||
@@ -29,9 +29,11 @@ from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import msgspec
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
PUSH_REG_NOTIF_PREFIX,
|
||||
NixlAgentMetadata,
|
||||
NixlConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
|
||||
@@ -512,7 +514,7 @@ class TestPushWriterNotifs:
|
||||
# Pretend the writer thread already forwarded a completion notif
|
||||
# for a request whose KV is being received.
|
||||
request_id = "req-recv-1"
|
||||
w._recving_metadata[request_id] = MagicMock()
|
||||
w._recving_metadata[request_id] = MagicMock(pp_size=1)
|
||||
# Compose the standard completion notif: req_id:tp_size.
|
||||
notif_msg = f"{request_id}:1".encode()
|
||||
w._pending_completion_notifs.put(notif_msg)
|
||||
@@ -813,3 +815,177 @@ class TestPushWriterNegative:
|
||||
assert w._reqs_to_send[rid] >= now
|
||||
# Unknown request must not be inserted by the heartbeat path.
|
||||
assert "req-unknown" not in w._reqs_to_send
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- #
|
||||
# Pipeline-parallel producer (push-mode PP-disagg) #
|
||||
# ----------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestPushPipelineParallel:
|
||||
"""PP-sharded producer: per-stage completion counting, pp_size plumbing,
|
||||
and remote-region slicing."""
|
||||
|
||||
def test_completion_waits_for_one_notif_per_pp_stage(self):
|
||||
"""Each PP stage WRITEs its own layers and sends one notif; D must
|
||||
collect pp_size notifs before reporting the recv done."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.transfer_topo = MagicMock()
|
||||
request_id = "req-pp-2"
|
||||
w._recving_metadata[request_id] = MagicMock(pp_size=2)
|
||||
notif = f"{request_id}:1".encode()
|
||||
|
||||
# First stage: counted, not yet done.
|
||||
w._pending_completion_notifs.put(notif)
|
||||
assert w._get_new_notifs() == set()
|
||||
assert request_id not in w._recving_transfers
|
||||
assert w.consumer_notification_counts_by_req[request_id] == 1
|
||||
|
||||
# Second (final) stage: now reported done.
|
||||
w._pending_completion_notifs.put(notif)
|
||||
assert w._get_new_notifs() == set()
|
||||
assert request_id in w._recving_transfers
|
||||
assert request_id not in w.consumer_notification_counts_by_req
|
||||
|
||||
def test_req_meta_reads_pp_size_from_kv_transfer_params(self):
|
||||
"""D learns the producer's pp_size from kv_transfer_params (forwarded
|
||||
by the proxy) and defaults to 1 when absent."""
|
||||
metadata = NixlConnectorMetadata()
|
||||
params = {
|
||||
"remote_block_ids": ([0],),
|
||||
"remote_engine_id": "p-engine",
|
||||
"remote_request_id": "p-req",
|
||||
"remote_host": "localhost",
|
||||
"remote_port": 1234,
|
||||
"tp_size": 1,
|
||||
"pp_size": 2,
|
||||
}
|
||||
metadata.add_new_req_to_recv("req", ([0],), params)
|
||||
assert metadata.reqs_to_recv["req"].pp_size == 2
|
||||
|
||||
params.pop("pp_size")
|
||||
metadata.add_new_req_to_recv("req-default", ([0],), params)
|
||||
assert metadata.reqs_to_recv["req-default"].pp_size == 1
|
||||
|
||||
def test_add_remote_agent_slices_remote_regions_to_local_pp_window(self):
|
||||
"""With PP>1 the producer registered regions for the full model, but
|
||||
this worker holds only a contiguous layer slice; add_remote_agent
|
||||
trims the remote region list to [offset : offset + num_local_regions]
|
||||
before building descriptors. We stop right after the slice via a
|
||||
sentinel on the next collaborator call."""
|
||||
block_len = 4096 * 16
|
||||
w = _StubWriterWorker.fresh() # seeds writer-thread state for teardown
|
||||
w.pp_size = 2
|
||||
w._remote_region_offset = 2 # this worker owns layers [2, 4)
|
||||
w.block_len_per_layer = [block_len, block_len] # 2 local layers
|
||||
w.nixl_wrapper = MagicMock()
|
||||
|
||||
class _StopAfterSlice(RuntimeError):
|
||||
pass
|
||||
|
||||
w.transfer_topo = MagicMock()
|
||||
w.transfer_topo.register_remote_engine.side_effect = _StopAfterSlice()
|
||||
|
||||
meta = NixlAgentMetadata(
|
||||
engine_id="p-engine",
|
||||
agent_metadata=b"agent",
|
||||
kv_caches_base_addr=[10, 11, 12, 13], # full 4-layer model
|
||||
device_id=0,
|
||||
num_blocks=4,
|
||||
block_lens=[block_len] * 4,
|
||||
kv_cache_layout="HND",
|
||||
block_size=16,
|
||||
ssm_sizes=(0, 0),
|
||||
attn_backend_name="FLASH_ATTN",
|
||||
physical_blocks_per_logical_kv_block=1,
|
||||
)
|
||||
|
||||
with pytest.raises(_StopAfterSlice):
|
||||
w.add_remote_agent(meta, remote_tp_rank=0, remote_tp_size=1)
|
||||
|
||||
# Sliced down to this worker's layer window (regions [2:4]).
|
||||
assert meta.kv_caches_base_addr == [12, 13]
|
||||
assert meta.block_lens == [block_len, block_len]
|
||||
|
||||
|
||||
class TestPushWriterMlaReplication:
|
||||
"""MLA latent KV is replicated across D's TP ranks, so when D_TP > P_TP
|
||||
(``tp_ratio < 0``) the producer must *WRITE* the latent into every D rank
|
||||
it handshook -- not write one and merely notify the rest, which would
|
||||
leave the un-written ranks decoding against stale KV."""
|
||||
|
||||
@staticmethod
|
||||
def _mla_worker_writing_to(d_ranks):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
TPMapping,
|
||||
)
|
||||
|
||||
engine_id = "decode-engine"
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.use_mla = True
|
||||
w.nixl_wrapper = MagicMock()
|
||||
w.transfer_topo = MagicMock()
|
||||
w.transfer_topo.get_engine_info.return_value = SimpleNamespace(
|
||||
remote_tp_size=len(d_ranks),
|
||||
remote_block_size=16,
|
||||
remote_physical_blocks_per_logical=1,
|
||||
)
|
||||
# D_TP > P_TP => negative ratio (one P rank feeds |ratio| D ranks).
|
||||
w.transfer_topo.tp_ratio.return_value = -len(d_ranks)
|
||||
# The tp-mapping collapses MLA to a single source rank (correct for
|
||||
# the pull/read direction); the push path must fan it back out.
|
||||
w.tp_mappings = {
|
||||
engine_id: TPMapping(
|
||||
source_ranks_per_group=((0,),),
|
||||
all_source_ranks=(0,),
|
||||
rank_to_attention_slot={0: 0},
|
||||
rank_offset_factor=0,
|
||||
)
|
||||
}
|
||||
w._logical_to_remote_kernel_block_ids = lambda block_ids, ratio: block_ids
|
||||
w.dst_xfer_side_handles = {engine_id: {r: 1000 + r for r in d_ranks}}
|
||||
w.src_xfer_handles_by_block_size = {16: 2000}
|
||||
w._remote_agents = {engine_id: {(0, r): f"agent-{r}" for r in d_ranks}}
|
||||
return w, engine_id
|
||||
|
||||
def test_mla_hetero_tp_writes_every_d_rank(self):
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
RemoteMeta,
|
||||
ReqMeta,
|
||||
)
|
||||
|
||||
w, engine_id = self._mla_worker_writing_to(d_ranks=(0, 1))
|
||||
|
||||
written: list[int] = []
|
||||
|
||||
def _fake_xfer(**kw):
|
||||
rank = kw["read_spec"].remote_rank
|
||||
written.append(rank)
|
||||
return 1000 + rank # in-flight handle, as the real method returns
|
||||
|
||||
w._xfer_blocks = _fake_xfer
|
||||
|
||||
meta = ReqMeta(
|
||||
local_block_ids=([100, 101],),
|
||||
local_physical_block_ids=([100, 101],),
|
||||
tp_size=1,
|
||||
remote=RemoteMeta(
|
||||
block_ids=([200, 201],),
|
||||
host="",
|
||||
port=0,
|
||||
engine_id=engine_id,
|
||||
request_id="d-req",
|
||||
),
|
||||
)
|
||||
|
||||
w._xfer_blocks_for_req(req_id="p-req", meta=meta)
|
||||
|
||||
# Every handshook D rank must receive a real WRITE of the latent...
|
||||
assert sorted(written) == [0, 1]
|
||||
# ...and no rank may be fobbed off with a bare completion notif.
|
||||
assert w.nixl_wrapper.send_notif.call_count == 0
|
||||
# All of the request's WRITE handles must be tracked together, so the
|
||||
# engine thread never sees a partial set and double-frees the request.
|
||||
assert sorted(w._sending_transfers["p-req"]) == [1000, 1001]
|
||||
|
||||
Reference in New Issue
Block a user