call sched_yield repeatedly when terminating threads

This should allow the threads we're terminating to more quickly
enter a safe state where they're allowed to exit.  On SMP systems,
we need to yield the signalling thread more times to increase the
probability the interrupted thread can run (and exit).

1 files changed, 20 insertions, 2 deletions

diff --git a/thrpool.c b/thrpool.c
index a0c0bc2..7d62b59 100644
--- a/thrpool.c
+++ b/thrpool.c
@@ -67,6 +67,24 @@ void mog_thr_test_quit(void)
 }
 
 /*
+ * sched_yield may migrate us to the same CPU as the task we're waiting
+ * on, so just keep yielding for every CPU we have as this throttles
+ * our ability to spam SIGURG.  This means the threads we're trying to
+ * gracefully kill off can finish their work and check their mog_do_quit
+ * flag sooner
+ */
+static void yield_all(void)
+{
+        static unsigned long nproc_all;
+        unsigned long i;
+
+        if (!nproc_all)
+                nproc_all = num_processors(NPROC_ALL) * 2;
+        for (i = 0; i < nproc_all; i++)
+                pthread_yield();
+}
+
+/*
  * we no longer rely on pthreads cancellation, so our explicit checks for
  * thread quitting requires us to continuously signal a thread for death
  * in case it enters a sleeping syscall (epoll_wait/kevent) immediately
@@ -81,7 +99,7 @@ static void poke(pthread_t thr, int sig)
          * we lower thread counts or shut down
          */
         while ((err = pthread_kill(thr, sig)) == 0)
-                sched_yield();
+                yield_all();
 
         assert(err == ESRCH && "pthread_kill() usage bug");
 }
@@ -97,7 +115,7 @@ thr_create_fail_retry(struct mog_thrpool *tp, size_t size,
                         syslog(LOG_ERR, "pthread_create: %m (tries: %lu)",
                                *nr_eagain);
                 }
-                sched_yield();
+                yield_all();
                 return true;
         } else {
                 errno = err;