From 20bcb2ccc3d3d38b0fc2f16c25cad74d8404d5bb Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 2 Mar 2013 10:46:12 +0000 Subject: fdmap: documentation for the FD-based memory allocation Avoiding heap allocations in common paths is important to high performance server design; document this important design decision. --- fdmap.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/fdmap.c b/fdmap.c index cdfadf5..c59157f 100644 --- a/fdmap.c +++ b/fdmap.c @@ -1,6 +1,33 @@ /* * Copyright (C) 2012-2013, Eric Wong * License: GPLv3 or later (see COPYING for details) + * + * File descriptor-based memory allocation. We have a fixed slot of + * 128 bytes for every file descriptor. Once a file descriptor is + * allocated by the OS, we use mog_fd_init()/mog_fd_get() to reserve + * userspace memory for that FD. We release that memory by calling + * close(2) (via mog_close() wrapper) in mog_fd_put(). + * + * mog_fd_get() is a simple offset lookup based on the file + * descriptor, so the "allocation" is simple. + * + * This memory is never returned to the kernel, but is bounded by + * the file descriptor limit (RLIMIT_NOFILE ("ulimit -n")) of the + * process. Allowing 20000 file descriptors will only use 2.5 MB + * of userspace memory. + * + * Any sane OS will try to keep file descriptors low and reuse + * low-numbered descriptors as they become available, reducing + * fragmentation from unused slots. We allocate aligned memory + * to 128 bytes (matching slot size). + * + * 128-byte alignment and slot size are used since it: + * a) is enough to hold per-client data in common cases without malloc() + * b) easy to align with cache line sizes of modern (200x-201x) CPUs, + * avoiding unnecessary cache flushing + * + * This 128-byte alignment will need to be expanded to 256 bytes when + * 128-bit general purpose CPUs become available. */ #include "cmogstored.h" #define FD_PAD_SIZE ((size_t)128) @@ -84,6 +111,10 @@ MOG_NOINLINE static struct mog_fd * grow_ref(size_t fd) return aref(fd); } +/* + * Look up a mog_fd structure based on fd. This means memory is reused + * by us just as FDs are reused by the kernel. + */ struct mog_fd *mog_fd_get(int fd) { assert(fd >= 0 && "FD is negative"); @@ -113,6 +144,10 @@ static inline void mfd_expiring_unlock(struct mog_fd *mfd) CHECK(int, 0, pthread_spin_unlock(&mfd->expiring)); } +/* + * Releases the memory used by mfd and releases the file descriptor + * back to the OS. mfd is unusable after this. + */ void mog_fd_put(struct mog_fd *mfd) { int fd = mfd->fd; @@ -125,6 +160,7 @@ void mog_fd_put(struct mog_fd *mfd) mfd->fd_type = MOG_FD_TYPE_UNUSED; mfd_expiring_unlock(mfd); mog_close(fd); + /* mog_fd_get(fd) may be called here in another thread */ } /* called during shutdown, no other threads are running when this is called */ -- cgit v1.2.3-24-ge0c7