From 02206cff4696014ec69019778d50b266e7f6924a Mon Sep 17 00:00:00 2001
From: Kawrakow <iwankawrakow@gmail.com>
Date: Thu, 11 Dec 2025 07:04:44 +0100
Subject: [PATCH] Reduce back-end syncs (#1049)

* Reduce backend synchronization calls

* Also this

---------

Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
---
 ggml/src/ggml-backend.cpp | 46 ++++++++++++++++++++++++++++-----------
 1 file changed, 33 insertions(+), 13 deletions(-)

diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
index f7c1593b..76d14127 100644
--- a/ggml/src/ggml-backend.cpp
+++ b/ggml/src/ggml-backend.cpp
@@ -12,6 +12,7 @@
 #include <string>
 #include <vector>
 #include <set>
+#include <array>
 
 #define IK_PRINT_TIMING 0
 
@@ -1879,6 +1880,8 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
     std::vector<uint32_t> unique_ids;
     ggml_tensor * last_ids_tensor = nullptr;
 
+    std::array<bool, GGML_SCHED_MAX_BACKENDS> needs_sync{{true}};
+
     for (int i = 0; i < sched->n_splits; i++) {
 #if IK_PRINT_TIMING
         int64_t tim1 = ggml_time_us();
@@ -1897,18 +1900,24 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
 
             if (input->flags & GGML_TENSOR_FLAG_INPUT) {
                 // inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done
-                if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
-                    ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);
-                } else {
-                    ggml_backend_synchronize(split_backend);
+                if (needs_sync[split_backend_id]) {
+                    if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
+                        ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);
+                    } else {
+                        ggml_backend_synchronize(split_backend);
+                    }
+                    needs_sync[split_backend_id] = false;
                 }
                 ggml_backend_tensor_copy(input, input_cpy);
             } else {
                 // wait for the split backend to finish using the input before overwriting it
-                if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
-                    ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]);
-                } else {
-                    ggml_backend_synchronize(split_backend);
+                if (needs_sync[split_backend_id]) {
+                    if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
+                        ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]);
+                    } else {
+                        ggml_backend_synchronize(split_backend);
+                    }
+                    needs_sync[split_backend_id] = false;
                 }
 
                 ggml_tensor * node = split->graph.nodes[0];
@@ -1944,6 +1953,7 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
                         ggml_backend_tensor_get_async(ids_backend, ids_tensor, ids.data(), 0, ggml_nbytes(ids_tensor));
 
                         ggml_backend_synchronize(ids_backend);
+                        needs_sync[tensor_backend_id(ids_tensor)] = false;
 
                         unique_ids.resize((n_expert + 31)/32);
                         std::memset(unique_ids.data(), 0, unique_ids.size()*sizeof(uint32_t));
@@ -2000,17 +2010,27 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
                 // try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events
                 // TODO: add public function to facilitate this, since applications do not have direct access to the backend interface
                 if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) {
-                    ggml_backend_synchronize(input_backend);
-                    if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
-                        ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);
-                    } else {
-                        ggml_backend_synchronize(split_backend);
+                    int input_backend_id = tensor_backend_id(input);
+                    if (needs_sync[input_backend_id]) {
+                        ggml_backend_synchronize(input_backend);
+                        needs_sync[input_backend_id] = false;
+                    }
+                    if (needs_sync[split_backend_id]) {
+                        if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
+                            ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);
+                        } else {
+                            ggml_backend_synchronize(split_backend);
+                        }
+                        needs_sync[split_backend_id] = false;
                     }
                     ggml_backend_tensor_copy(input, input_cpy);
                 }
             }
         }
 
+        if (split->n_inputs > 0) {
+            needs_sync[split_backend_id] = true;
+        }
         if (!sched->callback_eval) {
 #if IK_PRINT_TIMING
             int64_t tim2 = ggml_time_us();