diff --git a/ggml/src/ggml-cuda.cu b/ggml/src/ggml-cuda.cu
index 93a71c15..99d3091a 100644
--- a/ggml/src/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda.cu
@@ -512,8 +512,9 @@ ggml_backend_cuda_context::ggml_backend_cuda_context(int device) :
     auto info = const_cast<ggml_cuda_device_info*>(&ggml_cuda_info());
     if (info->all_ctx[device]) {
         GGML_CUDA_LOG_WARN("%s: a context for device %d already exists?\n", __func__, device);
+    } else{
+        info->all_ctx[device] = this;
     }
-    info->all_ctx[device] = this;
 }
 
 ggml_backend_cuda_context::~ggml_backend_cuda_context() {
@@ -527,9 +528,6 @@ ggml_backend_cuda_context::~ggml_backend_cuda_context() {
     std::unique_lock<std::mutex> lock(ggml_cuda_lock);
     ggml_cuda_lock_cv.wait(lock, []{ return ggml_cuda_lock_counter == 0; });
 
-    auto info = const_cast<ggml_cuda_device_info*>(&ggml_cuda_info());
-    info->all_ctx[this->device] = nullptr;
-
     if (copy_event != nullptr) {
         CUDA_CHECK(cudaEventDestroy(copy_event));
     }