[ROCm] Remove stale SDPA and skinny GEMM workarounds (#50907)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
4f2f31b82e
commit
1ab2801dde
@@ -6420,6 +6420,21 @@ class GPUModelRunner(
|
||||
logits,
|
||||
dummy_metadata,
|
||||
)
|
||||
# All-greedy is a separate Triton specialization from the
|
||||
# mixed-sampling path above. Compile and exercise it before serving.
|
||||
all_greedy_metadata = replace(
|
||||
dummy_metadata,
|
||||
temperature=None,
|
||||
all_greedy=True,
|
||||
all_random=False,
|
||||
)
|
||||
self.rejection_sampler(
|
||||
dummy_spec_decode_metadata,
|
||||
draft_probs,
|
||||
logits,
|
||||
all_greedy_metadata,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
return sampler_output
|
||||
|
||||
def _dummy_pooler_run_task(
|
||||
@@ -6615,16 +6630,26 @@ class GPUModelRunner(
|
||||
@staticmethod
|
||||
@contextmanager
|
||||
def _freeze_gc():
|
||||
gc_was_enabled = gc.isenabled()
|
||||
gc.collect()
|
||||
should_freeze = not envs.VLLM_ENABLE_CUDAGRAPH_GC
|
||||
if should_freeze:
|
||||
gc.freeze()
|
||||
# A Triton kernel finalized during stream capture unloads its
|
||||
# module and invalidates the captured graph.
|
||||
gc.disable()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if should_freeze:
|
||||
gc.unfreeze()
|
||||
gc.collect()
|
||||
try:
|
||||
gc.unfreeze()
|
||||
gc.collect()
|
||||
finally:
|
||||
if gc_was_enabled:
|
||||
gc.enable()
|
||||
else:
|
||||
gc.disable()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Release GPU tensors (model weights, KV caches, workspace) so that
|
||||
|
||||
Reference in New Issue
Block a user