queue: active clients maintain thread affinity

We want to avoid global resources like the active queue
as much as possible.

Unnecesarly bouncing of clients between different threads
and contention for the active queue lock hurts concurrency.
This contention is witnessed when parallel MD5 requests
are serviced during parallel fsck runs.

1 files changed, 8 insertions, 6 deletions

diff --git a/http.c b/http.c
index d3db941..98e3ed0 100644
--- a/http.c
+++ b/http.c
@@ -189,19 +189,20 @@ err400:
         return MOG_NEXT_CLOSE;
 }
 
-void mog_http_queue_step(struct mog_fd *mfd)
+struct mog_fd *mog_http_queue_step(struct mog_fd *mfd)
 {
         struct mog_queue *q = mfd->as.http.svc->queue;
 
         /* centralize all queue transitions here: */
         switch (http_queue_step(mfd)) {
-        case MOG_NEXT_CLOSE: http_close(mfd); return;
-        case MOG_NEXT_ACTIVE: mog_activeq_push(q, mfd); return;
-        case MOG_NEXT_WAIT_RD: mog_idleq_push(q, mfd, MOG_QEV_RD); return;
-        case MOG_NEXT_WAIT_WR: mog_idleq_push(q, mfd, MOG_QEV_WR); return;
+        case MOG_NEXT_CLOSE: http_close(mfd); break;
+        case MOG_NEXT_ACTIVE: return mfd;
+        case MOG_NEXT_WAIT_RD: mog_idleq_push(q, mfd, MOG_QEV_RD); break;
+        case MOG_NEXT_WAIT_WR: mog_idleq_push(q, mfd, MOG_QEV_WR); break;
         case MOG_NEXT_IGNORE:
                 assert(0 && "refused to put HTTP client into ignore state");
         }
+        return NULL;
 }
 
 /* called during graceful shutdown instead of mog_http_queue_step */
@@ -236,7 +237,8 @@ void mog_http_post_accept(int fd, struct mog_svc *svc)
         struct mog_http *http = &mfd->as.http;
 
         mog_http_init(http, svc);
-        mog_http_queue_step(mfd);
+        if (mog_http_queue_step(mfd))
+                mog_activeq_push(svc->queue, mfd);
 }
 
 /*