fdmap: documentation for the FD-based memory allocation

Avoiding heap allocations in common paths is important
to high performance server design; document this important
design decision.

1 files changed, 36 insertions, 0 deletions

diff --git a/fdmap.c b/fdmap.c
index cdfadf5..c59157f 100644
--- a/fdmap.c
+++ b/fdmap.c
@@ -1,6 +1,33 @@
 /*
  * Copyright (C) 2012-2013, Eric Wong <normalperson@yhbt.net>
  * License: GPLv3 or later (see COPYING for details)
+ *
+ * File descriptor-based memory allocation.  We have a fixed slot of
+ * 128 bytes for every file descriptor.  Once a file descriptor is
+ * allocated by the OS, we use mog_fd_init()/mog_fd_get() to reserve
+ * userspace memory for that FD.  We release that memory by calling
+ * close(2) (via mog_close() wrapper) in mog_fd_put().
+ *
+ * mog_fd_get() is a simple offset lookup based on the file
+ * descriptor, so the "allocation" is simple.
+ *
+ * This memory is never returned to the kernel, but is bounded by
+ * the file descriptor limit (RLIMIT_NOFILE ("ulimit -n")) of the
+ * process.  Allowing 20000 file descriptors will only use 2.5 MB
+ * of userspace memory.
+ *
+ * Any sane OS will try to keep file descriptors low and reuse
+ * low-numbered descriptors as they become available, reducing
+ * fragmentation from unused slots.  We allocate aligned memory
+ * to 128 bytes (matching slot size).
+ *
+ * 128-byte alignment and slot size are used since it:
+ * a) is enough to hold per-client data in common cases without malloc()
+ * b) easy to align with cache line sizes of modern (200x-201x) CPUs,
+ *    avoiding unnecessary cache flushing
+ *
+ * This 128-byte alignment will need to be expanded to 256 bytes when
+ * 128-bit general purpose CPUs become available.
  */
 #include "cmogstored.h"
 #define FD_PAD_SIZE ((size_t)128)
@@ -84,6 +111,10 @@ MOG_NOINLINE static struct mog_fd * grow_ref(size_t fd)
         return aref(fd);
 }
 
+/*
+ * Look up a mog_fd structure based on fd.  This means memory is reused
+ * by us just as FDs are reused by the kernel.
+ */
 struct mog_fd *mog_fd_get(int fd)
 {
         assert(fd >= 0 && "FD is negative");
@@ -113,6 +144,10 @@ static inline void mfd_expiring_unlock(struct mog_fd *mfd)
         CHECK(int, 0, pthread_spin_unlock(&mfd->expiring));
 }
 
+/*
+ * Releases the memory used by mfd and releases the file descriptor
+ * back to the OS.  mfd is unusable after this.
+ */
 void mog_fd_put(struct mog_fd *mfd)
 {
         int fd = mfd->fd;
@@ -125,6 +160,7 @@ void mog_fd_put(struct mog_fd *mfd)
         mfd->fd_type = MOG_FD_TYPE_UNUSED;
         mfd_expiring_unlock(mfd);
         mog_close(fd);
+        /* mog_fd_get(fd) may be called here in another thread */
 }
 
 /* called during shutdown, no other threads are running when this is called */