diff --git a/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh b/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh index 9ce225c3a4..9732f46737 100755 --- a/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh +++ b/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh @@ -28,6 +28,7 @@ hybrid_ssm_configs=( # GDN (Qwen3.5) "VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B" "VLLM_SSM_CONV_STATE_LAYOUT=DS PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B" + "VLLM_SSM_CONV_STATE_LAYOUT=DS ENFORCE_EAGER=0 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B VLLM_SERVE_EXTRA_ARGS=--spec-method,mtp,--spec-tokens,1" # Mamba1 (Jamba) "VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ai21labs/AI21-Jamba2-3B VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192" ) diff --git a/tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh b/tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh index 0e7f6af7e3..9ae15ad0e8 100755 --- a/tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh +++ b/tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh @@ -76,6 +76,7 @@ DECODER_TP_SIZE=${DECODER_TP_SIZE:-1} GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.2} PREFILL_BLOCK_SIZE=${PREFILL_BLOCK_SIZE:-128} DECODE_BLOCK_SIZE=${DECODE_BLOCK_SIZE:-128} +ENFORCE_EAGER=${ENFORCE_EAGER:-1} # Comma-separated extra args for vllm serve (e.g. --max-model-len,2048) VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-} @@ -157,11 +158,13 @@ run_tests_for_model() { VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \ vllm serve $model_name \ --port $PORT \ - --enforce-eager \ --block-size ${PREFILL_BLOCK_SIZE} \ --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \ --tensor-parallel-size $PREFILLER_TP_SIZE \ --kv-transfer-config '$KV_CONFIG_P'" + if [[ "$ENFORCE_EAGER" == "1" ]]; then + BASE_CMD="${BASE_CMD} --enforce-eager" + fi if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS" for arg in "${extra_args[@]}"; do @@ -206,10 +209,12 @@ run_tests_for_model() { VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \ vllm serve $model_name \ --port $PORT \ - --enforce-eager \ --block-size ${DECODE_BLOCK_SIZE} \ --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \ --kv-transfer-config '$KV_CONFIG_D'" + if [[ "$ENFORCE_EAGER" == "1" ]]; then + BASE_CMD="${BASE_CMD} --enforce-eager" + fi if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS" for arg in "${extra_args[@]}"; do diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index 006e11e450..64c1096dfb 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -340,10 +340,12 @@ def _combine_sampled_and_draft_tokens_kernel( # Handling prefill tokens. No sampled or draft tokens. return - if NUM_NEW_SAMPLED_TOKENS > 0: + # Keep prompt-tail slots intact; only rewrite generated-token slots. + first_logit_seq_pos = seq_len - num_logits + if NUM_NEW_SAMPLED_TOKENS > 0 and first_logit_seq_pos >= prefill_len: # Write the last sampled token ID to input_ids. last_token_id = tl.load(last_sampled_tokens_ptr + req_state_idx) - tl.store(input_ids_ptr + query_end - num_logits, last_token_id) + tl.store(input_ids_ptr + logits_start, last_token_id) # Write the draft tokens (if any) to input_ids. if num_draft_tokens > 0: diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index cf6ab0ac06..26439a9858 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -255,11 +255,16 @@ class MambaHybridModelState(DefaultModelState): # GDN uses >= 0 to select spec-decode rows, so non-decode rows # need the -1 sentinel rather than a raw zero draft count. num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32) - if input_batch.num_draft_tokens_per_req is not None: - has_draft_tokens = input_batch.num_draft_tokens_per_req > 0 - spec_decode_mask = has_draft_tokens & ~input_batch.is_prefilling_np + num_draft_tokens_per_req = input_batch.num_draft_tokens_per_req + if num_draft_tokens_per_req is not None: + # A row is a spec-decode row only when its whole prompt is already + # computed, i.e. exactly one non-draft (decode) token is scheduled. + is_decode = ( + input_batch.num_scheduled_tokens == num_draft_tokens_per_req + 1 + ) + spec_decode_mask = (num_draft_tokens_per_req > 0) & is_decode num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where( - spec_decode_mask, input_batch.num_draft_tokens_per_req, -1 + spec_decode_mask, num_draft_tokens_per_req, -1 ) num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np) diff --git a/vllm/v1/worker/gpu/states.py b/vllm/v1/worker/gpu/states.py index be7bae7f17..7f0ae33c80 100644 --- a/vllm/v1/worker/gpu/states.py +++ b/vllm/v1/worker/gpu/states.py @@ -110,16 +110,6 @@ class RequestState: self.num_computed_tokens_np[req_idx] = num_computed_tokens self.num_computed_tokens.stage_write_elem(req_idx, num_computed_tokens) - if 0 < num_computed_tokens <= prefill_len: - # For PD disagg or resumed requests: set last_sampled to the last - # computed token so the first decode step gets the right input_id. - # For fresh prefill requests (num_computed_tokens == 0) the tensor - # is not read by combine_sampled_and_draft_tokens so we skip the - # write. Use a slice assignment rather than scalar indexing so the - # write is dispatched through fill_ without a host/device sync. - self.last_sampled_tokens[req_idx : req_idx + 1] = all_token_ids[ - num_computed_tokens - 1 - ] self.draft_tokens[req_idx].zero_() def apply_staged_writes(self) -> None: diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 5cf3f5728b..e9b23f1c61 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -2208,10 +2208,7 @@ class GPUModelRunner( req_idx = self.input_batch.req_id_to_index[req_id] draft_len = len(draft_token_ids) num_draft_tokens[req_idx] = draft_len - if ( - self.input_batch.num_computed_tokens_cpu[req_idx] - >= self.input_batch.num_prompt_tokens[req_idx] - ): + if num_scheduled_tokens[req_idx] == draft_len + 1: num_decode_draft_tokens[req_idx] = draft_len spec_decode_metadata = self._calc_spec_decode_metadata( num_draft_tokens, cu_num_tokens