svc: increase responsiveness of graceful shutdown

By reducing the capacity of each ioq, we force each running worker
thread to yield the current client and hit an exit point
(epoll_wait/kqueue) sooner.

3 files changed, 22 insertions, 6 deletions

diff --git a/cmogstored.h b/cmogstored.h
index 182568c..8bbff90 100644
--- a/cmogstored.h
+++ b/cmogstored.h
@@ -398,6 +398,7 @@ void mog_svc_devstats_subscribe(struct mog_mgmt *);
 void mog_svc_dev_shutdown(void);
 void mog_mkusage_all(void);
 void mog_svc_dev_user_rescale(struct mog_svc *, size_t ndev_new);
+void mog_svc_dev_quit_prepare(struct mog_svc *);
 
 /* cloexec_detect.c */
 extern bool mog_cloexec_atomic;
diff --git a/exit.c b/exit.c
index 719d1bd..ec6f4d5 100644
--- a/exit.c
+++ b/exit.c
@@ -18,10 +18,11 @@ static void acceptor_quit(struct mog_fd **mfdp)
         }
 }
 
-static bool svc_quit_accept_i(void *svcptr, void *ignored)
+static bool svc_quit_i(void *svcptr, void *ignored)
 {
         struct mog_svc *svc = svcptr;
 
+        mog_svc_dev_quit_prepare(svc);
         acceptor_quit(&svc->mgmt_mfd);
         acceptor_quit(&svc->http_mfd);
         acceptor_quit(&svc->httpget_mfd);
@@ -42,7 +43,7 @@ _Noreturn void cmogstored_exit(void)
 {
         /* call atexit() handlers and make valgrind happy */
         setproctitle("cmogstored, shutting down");
-        mog_svc_each(svc_quit_accept_i, NULL);
+        mog_svc_each(svc_quit_i, NULL);
         mog_svc_dev_shutdown();
         mog_queue_stop(mog_notify_queue);
         mog_svc_dev_shutdown();
diff --git a/svc_dev.c b/svc_dev.c
index e66ca26..93e3c43 100644
--- a/svc_dev.c
+++ b/svc_dev.c
@@ -310,6 +310,14 @@ static void svc_rescale_warn_fix_capa(struct mog_svc *svc, size_t ndev_new)
         svc->thr_per_dev = 1;
 }
 
+static void mog_svc_dev_rescale_all(struct mog_svc *svc)
+{
+        /* iterate through each device of this svc */
+        CHECK(int, 0, pthread_mutex_lock(&svc->by_mog_devid_lock));
+        hash_do_for_each(svc->by_mog_devid, mog_dev_user_rescale_i, svc);
+        CHECK(int, 0, pthread_mutex_unlock(&svc->by_mog_devid_lock));
+}
+
 /* rescaling only happens in the main thread */
 void mog_svc_dev_user_rescale(struct mog_svc *svc, size_t ndev_new)
 {
@@ -319,9 +327,15 @@ void mog_svc_dev_user_rescale(struct mog_svc *svc, size_t ndev_new)
         svc->thr_per_dev = svc->user_set_aio_threads / ndev_new;
 
         svc_rescale_warn_fix_capa(svc, ndev_new);
+        mog_svc_dev_rescale_all(svc);
+}
 
-        /* iterate through each device of this svc */
-        CHECK(int, 0, pthread_mutex_lock(&svc->by_mog_devid_lock));
-        hash_do_for_each(svc->by_mog_devid, mog_dev_user_rescale_i, svc);
-        CHECK(int, 0, pthread_mutex_unlock(&svc->by_mog_devid_lock));
+/*
+ * this forces ioqs to detect contention and yield,
+ * leading to quicker shutdown
+ */
+void mog_svc_dev_quit_prepare(struct mog_svc *svc)
+{
+        svc->thr_per_dev = 1;
+        mog_svc_dev_rescale_all(svc);
 }